From 956683c6f1fbbaf1f0ceff786a64607b3804be9e Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Wed, 7 Jul 2021 09:30:55 -0700 Subject: [PATCH 01/10] add vcr 3.0.0 --- tools/vcrpy/LICENSE.txt | 7 + tools/vcrpy/MANIFEST.in | 6 + tools/vcrpy/PKG-INFO | 90 ++++ tools/vcrpy/README.rst | 64 +++ tools/vcrpy/setup.cfg | 7 + tools/vcrpy/setup.py | 69 +++ tools/vcrpy/tests/assertions.py | 19 + .../fixtures/migration/new_cassette.json | 35 ++ .../fixtures/migration/new_cassette.yaml | 20 + .../tests/fixtures/migration/not_cassette.txt | 1 + .../fixtures/migration/old_cassette.json | 34 ++ .../fixtures/migration/old_cassette.yaml | 18 + .../tests/fixtures/wild/domain_redirect.yaml | 146 +++++ tools/vcrpy/tests/integration/__init__.py | 0 .../vcrpy/tests/integration/aiohttp_utils.py | 43 ++ tools/vcrpy/tests/integration/test_aiohttp.py | 304 +++++++++++ tools/vcrpy/tests/integration/test_basic.py | 90 ++++ tools/vcrpy/tests/integration/test_boto.py | 84 +++ tools/vcrpy/tests/integration/test_boto3.py | 122 +++++ tools/vcrpy/tests/integration/test_config.py | 58 ++ .../vcrpy/tests/integration/test_disksaver.py | 55 ++ tools/vcrpy/tests/integration/test_filter.py | 130 +++++ tools/vcrpy/tests/integration/test_http | 22 + .../vcrpy/tests/integration/test_httplib2.py | 152 ++++++ tools/vcrpy/tests/integration/test_ignore.py | 67 +++ .../vcrpy/tests/integration/test_matchers.py | 107 ++++ .../vcrpy/tests/integration/test_multiple.py | 20 + tools/vcrpy/tests/integration/test_proxy.py | 59 +++ .../tests/integration/test_record_mode.py | 142 +++++ .../integration/test_register_matcher.py | 36 ++ .../integration/test_register_persister.py | 55 ++ .../integration/test_register_serializer.py | 33 ++ tools/vcrpy/tests/integration/test_request.py | 19 + .../vcrpy/tests/integration/test_requests.py | 301 +++++++++++ tools/vcrpy/tests/integration/test_stubs.py | 134 +++++ tools/vcrpy/tests/integration/test_tornado.py | 350 ++++++++++++ .../test_tornado_exception_can_be_caught.yaml | 62 +++ ...t_tornado_with_decorator_use_cassette.yaml | 53 ++ tools/vcrpy/tests/integration/test_urllib2.py | 144 +++++ tools/vcrpy/tests/integration/test_urllib3.py | 159 ++++++ tools/vcrpy/tests/integration/test_wild.py | 109 ++++ tools/vcrpy/tests/unit/test_cassettes.py | 370 +++++++++++++ tools/vcrpy/tests/unit/test_errors.py | 68 +++ tools/vcrpy/tests/unit/test_filters.py | 283 ++++++++++ .../vcrpy/tests/unit/test_json_serializer.py | 17 + tools/vcrpy/tests/unit/test_matchers.py | 274 ++++++++++ tools/vcrpy/tests/unit/test_migration.py | 47 ++ tools/vcrpy/tests/unit/test_persist.py | 30 ++ tools/vcrpy/tests/unit/test_request.py | 86 +++ tools/vcrpy/tests/unit/test_response.py | 103 ++++ tools/vcrpy/tests/unit/test_serialize.py | 119 +++++ tools/vcrpy/tests/unit/test_stubs.py | 17 + tools/vcrpy/tests/unit/test_vcr.py | 362 +++++++++++++ tools/vcrpy/tests/unit/test_vcr_import.py | 16 + tools/vcrpy/tox.ini | 71 +++ tools/vcrpy/vcr/__init__.py | 26 + tools/vcrpy/vcr/_handle_coroutine.py | 3 + tools/vcrpy/vcr/cassette.py | 360 +++++++++++++ tools/vcrpy/vcr/compat.py | 14 + tools/vcrpy/vcr/config.py | 254 +++++++++ tools/vcrpy/vcr/errors.py | 42 ++ tools/vcrpy/vcr/filters.py | 166 ++++++ tools/vcrpy/vcr/matchers.py | 142 +++++ tools/vcrpy/vcr/migration.py | 157 ++++++ tools/vcrpy/vcr/patch.py | 501 ++++++++++++++++++ tools/vcrpy/vcr/persisters/__init__.py | 0 tools/vcrpy/vcr/persisters/filesystem.py | 25 + tools/vcrpy/vcr/request.py | 139 +++++ tools/vcrpy/vcr/serialize.py | 58 ++ tools/vcrpy/vcr/serializers/__init__.py | 0 tools/vcrpy/vcr/serializers/compat.py | 77 +++ tools/vcrpy/vcr/serializers/jsonserializer.py | 29 + tools/vcrpy/vcr/serializers/yamlserializer.py | 15 + tools/vcrpy/vcr/stubs/__init__.py | 363 +++++++++++++ .../vcrpy/vcr/stubs/aiohttp_stubs/__init__.py | 209 ++++++++ tools/vcrpy/vcr/stubs/boto3_stubs.py | 44 ++ tools/vcrpy/vcr/stubs/boto_stubs.py | 8 + tools/vcrpy/vcr/stubs/compat.py | 44 ++ tools/vcrpy/vcr/stubs/httplib2_stubs.py | 60 +++ tools/vcrpy/vcr/stubs/requests_stubs.py | 19 + tools/vcrpy/vcr/stubs/tornado_stubs.py | 90 ++++ tools/vcrpy/vcr/stubs/urllib3_stubs.py | 15 + tools/vcrpy/vcr/util.py | 118 +++++ 83 files changed, 8267 insertions(+) create mode 100644 tools/vcrpy/LICENSE.txt create mode 100644 tools/vcrpy/MANIFEST.in create mode 100644 tools/vcrpy/PKG-INFO create mode 100644 tools/vcrpy/README.rst create mode 100644 tools/vcrpy/setup.cfg create mode 100644 tools/vcrpy/setup.py create mode 100644 tools/vcrpy/tests/assertions.py create mode 100644 tools/vcrpy/tests/fixtures/migration/new_cassette.json create mode 100644 tools/vcrpy/tests/fixtures/migration/new_cassette.yaml create mode 100644 tools/vcrpy/tests/fixtures/migration/not_cassette.txt create mode 100644 tools/vcrpy/tests/fixtures/migration/old_cassette.json create mode 100644 tools/vcrpy/tests/fixtures/migration/old_cassette.yaml create mode 100644 tools/vcrpy/tests/fixtures/wild/domain_redirect.yaml create mode 100644 tools/vcrpy/tests/integration/__init__.py create mode 100644 tools/vcrpy/tests/integration/aiohttp_utils.py create mode 100644 tools/vcrpy/tests/integration/test_aiohttp.py create mode 100644 tools/vcrpy/tests/integration/test_basic.py create mode 100644 tools/vcrpy/tests/integration/test_boto.py create mode 100644 tools/vcrpy/tests/integration/test_boto3.py create mode 100644 tools/vcrpy/tests/integration/test_config.py create mode 100644 tools/vcrpy/tests/integration/test_disksaver.py create mode 100644 tools/vcrpy/tests/integration/test_filter.py create mode 100644 tools/vcrpy/tests/integration/test_http create mode 100644 tools/vcrpy/tests/integration/test_httplib2.py create mode 100644 tools/vcrpy/tests/integration/test_ignore.py create mode 100644 tools/vcrpy/tests/integration/test_matchers.py create mode 100644 tools/vcrpy/tests/integration/test_multiple.py create mode 100644 tools/vcrpy/tests/integration/test_proxy.py create mode 100644 tools/vcrpy/tests/integration/test_record_mode.py create mode 100644 tools/vcrpy/tests/integration/test_register_matcher.py create mode 100644 tools/vcrpy/tests/integration/test_register_persister.py create mode 100644 tools/vcrpy/tests/integration/test_register_serializer.py create mode 100644 tools/vcrpy/tests/integration/test_request.py create mode 100644 tools/vcrpy/tests/integration/test_requests.py create mode 100644 tools/vcrpy/tests/integration/test_stubs.py create mode 100644 tools/vcrpy/tests/integration/test_tornado.py create mode 100644 tools/vcrpy/tests/integration/test_tornado_exception_can_be_caught.yaml create mode 100644 tools/vcrpy/tests/integration/test_tornado_with_decorator_use_cassette.yaml create mode 100644 tools/vcrpy/tests/integration/test_urllib2.py create mode 100644 tools/vcrpy/tests/integration/test_urllib3.py create mode 100644 tools/vcrpy/tests/integration/test_wild.py create mode 100644 tools/vcrpy/tests/unit/test_cassettes.py create mode 100644 tools/vcrpy/tests/unit/test_errors.py create mode 100644 tools/vcrpy/tests/unit/test_filters.py create mode 100644 tools/vcrpy/tests/unit/test_json_serializer.py create mode 100644 tools/vcrpy/tests/unit/test_matchers.py create mode 100644 tools/vcrpy/tests/unit/test_migration.py create mode 100644 tools/vcrpy/tests/unit/test_persist.py create mode 100644 tools/vcrpy/tests/unit/test_request.py create mode 100644 tools/vcrpy/tests/unit/test_response.py create mode 100644 tools/vcrpy/tests/unit/test_serialize.py create mode 100644 tools/vcrpy/tests/unit/test_stubs.py create mode 100644 tools/vcrpy/tests/unit/test_vcr.py create mode 100644 tools/vcrpy/tests/unit/test_vcr_import.py create mode 100644 tools/vcrpy/tox.ini create mode 100644 tools/vcrpy/vcr/__init__.py create mode 100644 tools/vcrpy/vcr/_handle_coroutine.py create mode 100644 tools/vcrpy/vcr/cassette.py create mode 100644 tools/vcrpy/vcr/compat.py create mode 100644 tools/vcrpy/vcr/config.py create mode 100644 tools/vcrpy/vcr/errors.py create mode 100644 tools/vcrpy/vcr/filters.py create mode 100644 tools/vcrpy/vcr/matchers.py create mode 100644 tools/vcrpy/vcr/migration.py create mode 100644 tools/vcrpy/vcr/patch.py create mode 100644 tools/vcrpy/vcr/persisters/__init__.py create mode 100644 tools/vcrpy/vcr/persisters/filesystem.py create mode 100644 tools/vcrpy/vcr/request.py create mode 100644 tools/vcrpy/vcr/serialize.py create mode 100644 tools/vcrpy/vcr/serializers/__init__.py create mode 100644 tools/vcrpy/vcr/serializers/compat.py create mode 100644 tools/vcrpy/vcr/serializers/jsonserializer.py create mode 100644 tools/vcrpy/vcr/serializers/yamlserializer.py create mode 100644 tools/vcrpy/vcr/stubs/__init__.py create mode 100644 tools/vcrpy/vcr/stubs/aiohttp_stubs/__init__.py create mode 100644 tools/vcrpy/vcr/stubs/boto3_stubs.py create mode 100644 tools/vcrpy/vcr/stubs/boto_stubs.py create mode 100644 tools/vcrpy/vcr/stubs/compat.py create mode 100644 tools/vcrpy/vcr/stubs/httplib2_stubs.py create mode 100644 tools/vcrpy/vcr/stubs/requests_stubs.py create mode 100644 tools/vcrpy/vcr/stubs/tornado_stubs.py create mode 100644 tools/vcrpy/vcr/stubs/urllib3_stubs.py create mode 100644 tools/vcrpy/vcr/util.py diff --git a/tools/vcrpy/LICENSE.txt b/tools/vcrpy/LICENSE.txt new file mode 100644 index 000000000000..d02308af0e12 --- /dev/null +++ b/tools/vcrpy/LICENSE.txt @@ -0,0 +1,7 @@ +Copyright (c) 2012-2015 Kevin McCarthy + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tools/vcrpy/MANIFEST.in b/tools/vcrpy/MANIFEST.in new file mode 100644 index 000000000000..9fc7449c539d --- /dev/null +++ b/tools/vcrpy/MANIFEST.in @@ -0,0 +1,6 @@ +include README.rst +include LICENSE.txt +include tox.ini +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] diff --git a/tools/vcrpy/PKG-INFO b/tools/vcrpy/PKG-INFO new file mode 100644 index 000000000000..257d812e0f94 --- /dev/null +++ b/tools/vcrpy/PKG-INFO @@ -0,0 +1,90 @@ +Metadata-Version: 1.2 +Name: vcrpy +Version: 3.0.0 +Summary: Automatically mock your HTTP interactions to simplify and speed up testing +Home-page: https://github.com/kevin1024/vcrpy +Author: Kevin McCarthy +Author-email: me@kevinmccarthy.org +License: MIT +Description: |PyPI| |Python versions| |Build Status| |CodeCov| |Gitter| |CodeStyleBlack| + + VCR.py + ====== + + .. image:: https://raw.github.com/kevin1024/vcrpy/master/vcr.png + :alt: vcr.py + + This is a Python version of `Ruby's VCR + library `__. + + Source code + https://github.com/kevin1024/vcrpy + + Documentation + https://vcrpy.readthedocs.io/ + + Rationale + --------- + + VCR.py simplifies and speeds up tests that make HTTP requests. The + first time you run code that is inside a VCR.py context manager or + decorated function, VCR.py records all HTTP interactions that take + place through the libraries it supports and serializes and writes them + to a flat file (in yaml format by default). This flat file is called a + cassette. When the relevant piece of code is executed again, VCR.py + will read the serialized requests and responses from the + aforementioned cassette file, and intercept any HTTP requests that it + recognizes from the original test run and return the responses that + corresponded to those requests. This means that the requests will not + actually result in HTTP traffic, which confers several benefits + including: + + - The ability to work offline + - Completely deterministic tests + - Increased test execution speed + + If the server you are testing against ever changes its API, all you need + to do is delete your existing cassette files, and run your tests again. + VCR.py will detect the absence of a cassette file and once again record + all HTTP interactions, which will update them to correspond to the new + API. + + License + ======= + + This library uses the MIT license. See `LICENSE.txt `__ for + more details + + .. |PyPI| image:: https://img.shields.io/pypi/v/vcrpy.svg + :target: https://pypi.python.org/pypi/vcrpy + .. |Python versions| image:: https://img.shields.io/pypi/pyversions/vcrpy.svg + :target: https://pypi.python.org/pypi/vcrpy + .. |Build Status| image:: https://secure.travis-ci.org/kevin1024/vcrpy.svg?branch=master + :target: http://travis-ci.org/kevin1024/vcrpy + .. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg + :alt: Join the chat at https://gitter.im/kevin1024/vcrpy + :target: https://gitter.im/kevin1024/vcrpy?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge + .. |CodeCov| image:: https://codecov.io/gh/kevin1024/vcrpy/branch/master/graph/badge.svg + :target: https://codecov.io/gh/kevin1024/vcrpy + :alt: Code Coverage Status + .. |CodeStyleBlack| image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + :alt: Code Style: black + +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Testing +Classifier: Topic :: Internet :: WWW/HTTP +Classifier: License :: OSI Approved :: MIT License +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.* diff --git a/tools/vcrpy/README.rst b/tools/vcrpy/README.rst new file mode 100644 index 000000000000..1794ad257421 --- /dev/null +++ b/tools/vcrpy/README.rst @@ -0,0 +1,64 @@ +|PyPI| |Python versions| |Build Status| |CodeCov| |Gitter| |CodeStyleBlack| + +VCR.py +====== + +.. image:: https://raw.github.com/kevin1024/vcrpy/master/vcr.png + :alt: vcr.py + +This is a Python version of `Ruby's VCR +library `__. + +Source code + https://github.com/kevin1024/vcrpy + +Documentation + https://vcrpy.readthedocs.io/ + +Rationale +--------- + +VCR.py simplifies and speeds up tests that make HTTP requests. The +first time you run code that is inside a VCR.py context manager or +decorated function, VCR.py records all HTTP interactions that take +place through the libraries it supports and serializes and writes them +to a flat file (in yaml format by default). This flat file is called a +cassette. When the relevant piece of code is executed again, VCR.py +will read the serialized requests and responses from the +aforementioned cassette file, and intercept any HTTP requests that it +recognizes from the original test run and return the responses that +corresponded to those requests. This means that the requests will not +actually result in HTTP traffic, which confers several benefits +including: + +- The ability to work offline +- Completely deterministic tests +- Increased test execution speed + +If the server you are testing against ever changes its API, all you need +to do is delete your existing cassette files, and run your tests again. +VCR.py will detect the absence of a cassette file and once again record +all HTTP interactions, which will update them to correspond to the new +API. + +License +======= + +This library uses the MIT license. See `LICENSE.txt `__ for +more details + +.. |PyPI| image:: https://img.shields.io/pypi/v/vcrpy.svg + :target: https://pypi.python.org/pypi/vcrpy +.. |Python versions| image:: https://img.shields.io/pypi/pyversions/vcrpy.svg + :target: https://pypi.python.org/pypi/vcrpy +.. |Build Status| image:: https://secure.travis-ci.org/kevin1024/vcrpy.svg?branch=master + :target: http://travis-ci.org/kevin1024/vcrpy +.. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg + :alt: Join the chat at https://gitter.im/kevin1024/vcrpy + :target: https://gitter.im/kevin1024/vcrpy?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge +.. |CodeCov| image:: https://codecov.io/gh/kevin1024/vcrpy/branch/master/graph/badge.svg + :target: https://codecov.io/gh/kevin1024/vcrpy + :alt: Code Coverage Status +.. |CodeStyleBlack| image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + :alt: Code Style: black diff --git a/tools/vcrpy/setup.cfg b/tools/vcrpy/setup.cfg new file mode 100644 index 000000000000..adf5ed72aa40 --- /dev/null +++ b/tools/vcrpy/setup.cfg @@ -0,0 +1,7 @@ +[bdist_wheel] +universal = 1 + +[egg_info] +tag_build = +tag_date = 0 + diff --git a/tools/vcrpy/setup.py b/tools/vcrpy/setup.py new file mode 100644 index 000000000000..9f21a8bd0ce4 --- /dev/null +++ b/tools/vcrpy/setup.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +import sys + +from setuptools import setup, find_packages +from setuptools.command.test import test as TestCommand + +long_description = open("README.rst", "r").read() + + +class PyTest(TestCommand): + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + # import here, cause outside the eggs aren't loaded + import pytest + + errno = pytest.main(self.test_args) + sys.exit(errno) + + +install_requires = [ + "PyYAML", + "wrapt", + "six>=1.5", + 'contextlib2; python_version=="2.7"', + 'mock; python_version=="2.7"', + 'yarl; python_version>="3.6"', + 'yarl<1.4; python_version=="3.5"', +] + +excluded_packages = ["tests*"] +if sys.version_info[0] == 2: + excluded_packages.append("vcr.stubs.aiohttp_stubs") + +setup( + name="vcrpy", + version="3.0.0", + description=("Automatically mock your HTTP interactions to simplify and " "speed up testing"), + long_description=long_description, + author="Kevin McCarthy", + author_email="me@kevinmccarthy.org", + url="https://github.com/kevin1024/vcrpy", + packages=find_packages(exclude=excluded_packages), + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", + install_requires=install_requires, + license="MIT", + tests_require=["pytest", "mock", "pytest-httpbin"], + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Software Development :: Testing", + "Topic :: Internet :: WWW/HTTP", + "License :: OSI Approved :: MIT License", + ], +) diff --git a/tools/vcrpy/tests/assertions.py b/tools/vcrpy/tests/assertions.py new file mode 100644 index 000000000000..7f1df8088642 --- /dev/null +++ b/tools/vcrpy/tests/assertions.py @@ -0,0 +1,19 @@ +import json + + +def assert_cassette_empty(cass): + assert len(cass) == 0 + assert cass.play_count == 0 + + +def assert_cassette_has_one_response(cass): + assert len(cass) == 1 + assert cass.play_count == 1 + + +def assert_is_json(a_string): + try: + json.loads(a_string.decode("utf-8")) + except Exception: + assert False + assert True diff --git a/tools/vcrpy/tests/fixtures/migration/new_cassette.json b/tools/vcrpy/tests/fixtures/migration/new_cassette.json new file mode 100644 index 000000000000..5526989b5f98 --- /dev/null +++ b/tools/vcrpy/tests/fixtures/migration/new_cassette.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "interactions": + [ + { + "request": { + "body": null, + "headers": { + "accept": ["*/*"], + "accept-encoding": ["gzip, deflate, compress"], + "user-agent": ["python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0"] + }, + "method": "GET", + "uri": "http://httpbin.org/ip" + }, + "response": { + "status": { + "message": "OK", + "code": 200 + }, + "headers": { + "access-control-allow-origin": ["*"], + "content-type": ["application/json"], + "date": ["Mon, 21 Apr 2014 23:13:40 GMT"], + "server": ["gunicorn/0.17.4"], + "content-length": ["32"], + "connection": ["keep-alive"] + }, + "body": { + "string": "{\n \"origin\": \"217.122.164.194\"\n}" + } + } + } + ] +} diff --git a/tools/vcrpy/tests/fixtures/migration/new_cassette.yaml b/tools/vcrpy/tests/fixtures/migration/new_cassette.yaml new file mode 100644 index 000000000000..e319dc8e1e4b --- /dev/null +++ b/tools/vcrpy/tests/fixtures/migration/new_cassette.yaml @@ -0,0 +1,20 @@ +version: 1 +interactions: +- request: + body: null + headers: + accept: ['*/*'] + accept-encoding: ['gzip, deflate, compress'] + user-agent: ['python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0'] + method: GET + uri: http://httpbin.org/ip + response: + body: {string: "{\n \"origin\": \"217.122.164.194\"\n}"} + headers: + access-control-allow-origin: ['*'] + content-type: [application/json] + date: ['Mon, 21 Apr 2014 23:06:09 GMT'] + server: [gunicorn/0.17.4] + content-length: ['32'] + connection: [keep-alive] + status: {code: 200, message: OK} diff --git a/tools/vcrpy/tests/fixtures/migration/not_cassette.txt b/tools/vcrpy/tests/fixtures/migration/not_cassette.txt new file mode 100644 index 000000000000..e1d9fc441308 --- /dev/null +++ b/tools/vcrpy/tests/fixtures/migration/not_cassette.txt @@ -0,0 +1 @@ +This is not a cassette diff --git a/tools/vcrpy/tests/fixtures/migration/old_cassette.json b/tools/vcrpy/tests/fixtures/migration/old_cassette.json new file mode 100644 index 000000000000..f6bfed1b5b1a --- /dev/null +++ b/tools/vcrpy/tests/fixtures/migration/old_cassette.json @@ -0,0 +1,34 @@ +[ + { + "request": { + "body": null, + "protocol": "http", + "method": "GET", + "headers": { + "accept-encoding": "gzip, deflate, compress", + "accept": "*/*", + "user-agent": "python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0" + }, + "host": "httpbin.org", + "path": "/ip", + "port": 80 + }, + "response": { + "status": { + "message": "OK", + "code": 200 + }, + "headers": [ + "access-control-allow-origin: *\r\n", + "content-type: application/json\r\n", + "date: Mon, 21 Apr 2014 23:13:40 GMT\r\n", + "server: gunicorn/0.17.4\r\n", + "content-length: 32\r\n", + "connection: keep-alive\r\n" + ], + "body": { + "string": "{\n \"origin\": \"217.122.164.194\"\n}" + } + } + } +] diff --git a/tools/vcrpy/tests/fixtures/migration/old_cassette.yaml b/tools/vcrpy/tests/fixtures/migration/old_cassette.yaml new file mode 100644 index 000000000000..cd1eacb2930f --- /dev/null +++ b/tools/vcrpy/tests/fixtures/migration/old_cassette.yaml @@ -0,0 +1,18 @@ +- request: !!python/object:vcr.request.Request + body: null + headers: !!python/object/apply:__builtin__.frozenset + - - !!python/tuple [accept-encoding, 'gzip, deflate, compress'] + - !!python/tuple [user-agent, python-requests/2.2.1 CPython/2.6.1 Darwin/10.8.0] + - !!python/tuple [accept, '*/*'] + host: httpbin.org + method: GET + path: /ip + port: 80 + protocol: http + response: + body: {string: !!python/unicode "{\n \"origin\": \"217.122.164.194\"\n}"} + headers: [!!python/unicode "access-control-allow-origin: *\r\n", !!python/unicode "content-type: + application/json\r\n", !!python/unicode "date: Mon, 21 Apr 2014 23:06:09 GMT\r\n", + !!python/unicode "server: gunicorn/0.17.4\r\n", !!python/unicode "content-length: + 32\r\n", !!python/unicode "connection: keep-alive\r\n"] + status: {code: 200, message: OK} diff --git a/tools/vcrpy/tests/fixtures/wild/domain_redirect.yaml b/tools/vcrpy/tests/fixtures/wild/domain_redirect.yaml new file mode 100644 index 000000000000..618babc18094 --- /dev/null +++ b/tools/vcrpy/tests/fixtures/wild/domain_redirect.yaml @@ -0,0 +1,146 @@ +version: 1 +interactions: +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate, compress'] + User-Agent: ['vcrpy-test'] + method: GET + uri: http://seomoz.org/ + response: + body: {string: ''} + headers: + Location: ['http://moz.com/'] + Server: ['BigIP'] + Connection: ['Keep-Alive'] + Content-Length: ['0'] + status: {code: 301, message: Moved Permanently} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate, compress'] + User-Agent: ['vcrpy-test'] + method: GET + uri: http://moz.com/ + response: + body: + string: !!binary | + H4sIAAAAAAAAA+08a3PbOJKfV78Co7nETo1I8SlKjuWUXxlnzpl4EieZuVTKBZKgyJgiGZKSbM9O + 1f2N+3v3S64bIClQkh3ntXu1tUlskiDQ3Wj0G2B2fzh6cXj+x9kxCctpvNfZ/UFR3kUBiUvy7Jg4 + 7/fILr4gXkyLYtxNUuVDAS+ViI3EZSguTpfENJmMuyzpwpgf3rHEj4L3irIEWcGDP58CeS9Yw0/A + +jSQSVnBwYZNE10BoShtMCGj/l4HSCijMmZ7r45fTNMbEhUkSRfkeXqjkldpUC5ozghNfHKYTqez + JCqvSZDm5ICVJcvJc5pfsjJKJipRcMxuXwDrdDq7U1ZS4oU0L1g57s7KQBl296rmsCwzhX2cRfNx + 93fl9b4C0DNaRm7MusRLk5IlMObZ8Zj5E9bzwjydsrG+cfih6K2cX2fy2JJdlX1kyeOGBkEC6QOU + OEouSc7icZfGMI+EljC2BAjQkGVx5AEtadLPi+Knq2kMr3BW4+7LV6+IoWpdEuYsGHeRjJ1+P2DM + L1T87c7yhOWql077wEs3TicCHSc6oTCH7jxiiyzNS4nUReSX4Vg3NA0miAsidfdZ4eVRhtRII96y + GHAwUqZiod4y4uUMJkFgFUlRLVuPTOvlgRWk8XUZeQWMSeOix5eUxjG5jBK/IGlQA0eYIYszcp3O + SMxonpAocdMZdJ8uF/st20KxiAvoXBFShowsmLtVkGlalGQeuTkFcF4jNoAjTYDxrALE8kLl7EFh + EbMkRe6Nu/2+7w1DPymmNzSbT1UvTmd+kAN9asLKPgg4K4u+GFEAo31YwOgmVz8UT1zfGHjGcDhy + mD/Qh5pv6czTKdNt2wsM3eqSC89PQDryGevu7VZAgARJIoryOmZFyFhZr/M9CeLj+pL8qF5RPPFH + pu/pTmDpph4wx3YcNhwarm0OXMv0NKOWOy6wMKBFomCPLK4AnSllOvNCJYIlU7KcAYuztGB+lxTR + DQPtt50r2/k84qMpnQDxAZ0jVJRepbpXbEfNkskTx7RsatsWBbIHTqC5emBZ2tA1zKFv6o62plj3 + odQxrhzjm1HqGJxSag9HI5caI83wDVcfjthgRH3NtzWdaiPD+yJKdd260lGAvhGtAIsTO3R9NkKy + KPOCkRkEOrVNyzNHwcDQTPaFxFpArPUNibUEsSPNZgPHd6ipGcwOLAbSQIPBgALVQ88X8to2YHnq + pmUh2a4kTf1MCDb0XE7Mo0magOLEK9YV6ECD2oyQbSnLo+BamesyeEOP02j+84nd/+OP/YkbXv4e + /TYNWf/t4mrm9G+Cm7OTydPzjx9ibbyB3EmaToC/RVQyhYOvVFnCEFyEbywrWZx55dHHP8wwu05s + 46k/O4eJ/WdwOYt+ObGHw99+O3z1egOCaTGnceSDrVY1me7B/mCo7ZsD0x5ZT48OrQNjdKTp1qFh + a/bAPDwS8xeQsjzNWF5eg5FYROiHd6jngYkuLyJfAqnbAxsUU5N80HJk4O5ksN7tEUNrODLMgWHZ + Fbq+CBI6u27qX9fBBbf4+EsB38GuiEu9y0mOLgL4FgPX5JYkjQowtcgFP5qTyEeRSNH9NZFKCkOC + OF3wXhgSAU4AUnWQYLnxjCkgJ3MKToYWJXZUkHgKXiVfIqlGNm+azjUGqVPe4JUoBMedNhQWGU1M + 4gGDKhxVZ1prl9QIzcV8QoRH74JLB2Fm0SQE3prAUwg2xl3w2BjIQLgTkiCKQfR/DIIA3AD4ywJi + qymgBDVg2/qjLgHIQM9zXTNUs2cY6vAUgPYs1TgZmurAUzTV6WmKDi81dcCv8BNrPVMdHjpDdQQ3 + Ts/BwVpvYKmDnuYpuoZDhtA2UC1FH6nDHhgjh0/iDYAOlYFqQjfVhn4GwAAkiom9VOcUaLB6MMY+ + 1TVV79nqCHrCX0ABI3g/vJ5oc8MBUI5qeZxGpAuoHfAr/ISaB2QAAoTCfwv8ugP060i1bld3Cr8D + CDB9HITgBKD6B9G8MTSgGonGmdWT6jXT83QdMfWgm644MCt+A+hsjthDLg6ADhOmYMB1qFjqMNQH + p0OcsamrVqhDg2nhjHrNX/ke57R5rnPAaJ7wVbwhz/lCGAPVEJgVm3Na1wDzCC9DoFZD9LDQNj71 + qkbeER5gVUd4gbZDhwPDFXaQlQi21yC4EbrfB6GUJLdPG4nvg8g3Dwmdy0KvGy3JrjrNlkkGdIcf + BS1LWwfiqO4DdnRKVsA0mtO28H0wTv7MQ2dRUpebFlCh7t5Z1SyRLUiPo2+AlIe4/YKlMlaY+Cm2 + fx+UTUgsozS7e02G9X3QinRkidHq7h1A03fiK4WcoZTR2d29fWy7E91ufxZvkDnJXtcCp0C4nIND + m8WxkqOFFea/JZ0inepsmMXS5Nfk/9jY+zKdYAiQpdkskycwgIRhCQScC+Gx0ccZKzBCwKQGmHMC + udNuP9qTlQzJz1NMIKcsmTWIOAbCfys85aI5eCOI7TLZzUhznzI/WtE06S1nRcyCcnWFWkQreTph + OceHCVB7qWVrsBG5gnHAKvxsb7coIZqc7N0mDALbOWSIyB5yMnORPWCWxDCikGOIAa7LEFNVTDwT + SKYxqbzEWgQXJZ7o7vazO+hdfQzz5ZJmGoE4tEynuszbCv9v1RIWhEsoeVYlvE1140lD6nJsW1BB + vvbeRBA28jz4ThXcg5kQ1DzkAS+mTGYR5PmQk+eQocesys5TgJS3U2ZkT86KdJZ7rFA3aOo9aGis + T/9jd++3h3SaPd7nlDxN89kUmZ5DpEQ+NjyBEDMEgooFJOwEsoYpAYWYAT+uCbvCaHKVElmJpTVZ + dlo1KQXYWi/8Ep10NuukAAja+IrffIE+VhA2aiTGaDxMFIFq1VfB5rZyRkkG0iTS+2qOVRYgni4g + ooTEJoupx8I0BljjrqBYVdWGLA5FKaY0jlciTRYzr1yBKKpgd4/ko1NeVCKQjcyw6oVd9uMYqzS8 + jidef2KU0OulTt9r0Ecqyd36CFA1Pq9WmzsD3U2alEA8iQvPCur7aq4/p7t90SJLJS7QnXLZqFHb + VdwaOqSTKOluJqp6t5TUIYQU6QRMi7B8S5RCW9qBWB+cV01c1V7diIyM5dXbAhgVSYzZnCtVcsrA + qEhpzOZUaRHCpCtbabdSu1nizvKi3JxDkXLBIPVrp1JSLDkAwQoKVpqyIoVW0wVTbcCWIAmCRoLe + gBQpJMtglDK7okn7jGinD9m8715DOs9SLGeAuMapAnfL3C7mld808eLIuxx3Lyb0o5rNinD73dYF + xBfeJTimpNzqkS1MeDFjxnus1REsXVRShm1o1feTBHjlsSmOef/o8Qqt6yX2ltxTWVhDS2aU3hhC + ZIpYI49mRc0Uwj0cZ9aKjQDeN3m7IXh6yfKE+dwf5wypocQV5fwFhdgTh+yByuUyIHALfrr0QjKh + ukyo2SK0pBPuvYA6vZapFn1/gCsjHjgTYC54F/CCLJ8DTWWLM5XvfcuWlWkgBjptcMv90Nyc3KwI + pJzb7EbTyeeUnqtKGUpEX4RThqbxApnrB7rtW66js5FlDjzdG7mO5Y4CPfCCkW9VJYCXOAgFwJUq + IRwSsgp61QUE01wWEPSRs1obXpvnurmoLATcygzwcsaSIkzLlkH4CjYsIYpisT1yLIu5zNYdwx26 + I50GZjDQ6Wik+4FvLkshoDP1zsSrBsaSAZruSCUUy1jlQD3PjbZMnhsIvyyZjTF5lpQ8v8TQqkUO + CFdoiKFWa6iwVahwEMpt2GCpt19QZ6bQPGF8o+UaRb3eSmEBOCMeOAk931yRWpVZa82ILotQXsi8 + Swhe+wEwscwjrKPWDClp7ZYmyGGgAoAo3IiBmfQuP98ALs3e0xrdxcmZMHnn9BJmTPAFMTXliF6T + c+zQGDhZXldmuIzOd0Ob8B0VCNJoDs4Uo7Ad3c6uAMWLHBhnfxKatYFPyyLDP4I7FS/OYe3b/Nnn + beRFQDArqjda11nU6LG43O7vq8orNx+3e/dWFVPuMocMJG0xv+WZW35jvQT6I3pXAYL4FJLzaQAh + NFoS+U0FMc1YIiJ5hVf/v9AHCyb3OWx8foM3DZMl297KfnnZuposTwtkN7egJcT8uC7gkPLrxo1g + wLYmbusWtrO2PquMlwMZe22ZeBiVZrV271bpoCzRS++5aV58a5cg6xQ/XSSt/F5G5KYbiva31ByQ + 4nZI0bakIM+QCCLPGluIPD1HUhoT2ozN9rh9vWGNJUQRwE1pYR9x36VH/KjwcFOAhMAR3l5vTkOw + UoSgKBAShkD8JARr64FtIbw6IfLmaQq2lyetOBLNodjzVslq+aD1dHsoyUffai++yjwg5EZmj6+y + GGlHsvmbtVKZRPBKwiBFAJXYfKkQ1QvzjxYjXnCtSy8oQsuzJRvEaHnioS6ecKUuJBHAMwmMgkJj + vLh2hoH3u2RMhJOzDHfkfIIFhLq44bM5i9MMY/lirfJ0T9GRqsvfQXxe1qWgjSLUvP2niBHcgDQU + fPP0Hy5Lv6SYfIQMxehQImSTJNV9IVbRepqm/QS2ZOpiNrIIU2548ghYzuUJ2bp2sEWInDgqs7E8 + Bg18ZFX0wJLfbPqlEiVtHnwHiWq2IDZK1O0bFJ8rURucZ7P5uh4xYDQRQkt7G3kRFWVEL5jlz3Xz + CtKupn/9BpbR73aqKJJnFDvD4QgiSJFQ7NiaxsNJAbJkU9wer2Mh3gB+KGP8TlTxOtVqFBBuT6ma + 5pM+jzteuB9gIgCJ77IvQWFtTtpfxwSjsnBbRecVBhk8iVkZ5c/y1UMHZ+f6c9t+1fTuLHuX4Wzq + JjSKX+exNKIhVdQ+IGtSBV+4GPksjvB8A+RvbODrrmk41Hb9geEPjYFmasw0PNt2zGDoeSa81Ea+ + 6oJqddaprTC+fnn6JdgHhsVMXXPdINCGmh5YvuMHpmvRge1Sm3mew4JRYHi3YeeL/Dm4A5CRsM+H + XWQxvWb5xdxQNbVYBE8MTTcVzVE08yGdlekZvB4HNC7YQ1Gaqh64UQD/jMV2N2YvktOU+mNMSR+K + WsZhCjozdlxnNKAPWeJzGTlgIZ1H0O6zgM7i8mEwi2ORNx9FBQVAFYzQh5V8YB9ABvLAPhoH8bxp + EoKLrY6hNa0om9gW+nLfGf6GRmDGA3P/gfEU/m1kCbQvFwQffGvowmroA2pbTuDbvslMy6POyPNc + wzRdDSTE04e4JA02rl6ITzeG2kMejh1VYjzWdVvVHiKzD7ilqfgmZgv5/uKsedVuc2m+bHiTxrNp + PaiMYpT39uz4ot4xsXtL+ods8sB8yusbFx6I2UXO8GSV/8A8AgNyBXbjIVZDg4BBKPqKsctKMLjl + WCPrXkz/Wj2YgZmm/hFPpRpNEPJsK4Z+bug7mrljDf+Lm5CU26vORjN6geFKh5tSfO/FcNk5MpzD + o4PBsbJ/PDhSdN0LlNHgYKhYlmXbJjAFPGdjaiF+x8Xe4elz29hmKYT4IBM7OYO0D6b/WLbLYEAz + mtNppzorlc4jmE61b/At1BrjD46hjYjGcbpYqmODsj4mKg8h0hBxfJR6EGQV3U6zlbKg18Utw9yJ + h7ahwfCjxv/cQtcCD7c2fdOMfryVHj7/Oc0lOv4FLVjnFm3q/IvYsM66EVub2j/Fit3G+K8yY0tJ + 5pPu8IrRt9DyzopC1/XiNaVttJVUijmuFLLT6NO/FenfivT/XZFqd7HmyvEDmLvdsq5pD271yvxl + t7P+YcyVAovgXS7onClcU7od7qtkJ8X5D1cRa+ztJmn1kYeU6Ehft+CxmE6THBGeHO3260HVRo/8 + uykL1GeKLwL4hVWD1RqtnDXWfUROKp7qvLF6giz6+vYa+n3OEG/cgyCtI8MrNRM8cawIxPz4RfvA + yl0bmc6Go8nLU3AxiIRSRNMsZvyspnTb4kQ98r5nH/iZrD5nCua9h+KGvC5Wjzd8HlhRat17yarz + N00Z9MtBgocpYdkA6ll195Xwqg2kPs2i7t7+2bOvnDHLp4WS5dGcerD45/hIeKGInInGdfjtU5K7 + mVQYMzYek6h30E/T9BJrn3jobFkJhVUswAm1zrmtbP9j4cJPWZH873//T4lfEaBpkwb2sAJF8DzB + Fh4nQKnDk0T3OITGv16ZgmPEnSXpAetrIN2RKF/yWpyf/iCxINvEger8z2GaXfNDoQQcdHb9mEBs + YFXHL3rkWeKpBM86vcQuBRZK8dSBL5fk7lQ5s9qHbZ0lzJalzOqcYcN1VgJXXEYCsO2JX/yw5HJr + Fusnq4WKKmKP464j1p84FHPnF4qrpkhgU/ICFfHVqxVbJJjziRNTdxJTfbNS03Ab/qobqIS4+eZ0 + LBYLNaAec0EpPkVM3a+797S6+y7k4GYYBEsJJwdL4VnxhFvaMbcIE3BgxsgZWNqIP5f55ZgmrLyY + TS4meQZqeRv9NeDu3im/w2No35L+AiaQxbNCFd9Pcfp13bAtS3NszdIse6BZw4E56IqPvbKZCzYi + 5M56I8ECDoLE03x4/9M353gWod9kRVmvfvhx3TFX9DR9wY3Ut58k6NaDsKtbyELPVyKWr/kk9gY/ + hmWWMRyZ1PXoIDCCoRMYrm1rBmVDI9BHrnPbx7A1ZvH1OP8u9QOdU9GKZ+qfcyueBNFExS2Eop7v + mLzD6v2FKKC/fyx39FmGdj3xwKLzjrQUGysQ+1146JdYt1edqG2GzvIcK8tgnskSx59/Pe50/mMb + O0TgQh41TxCAzkD2JtHNNIVW6ete6Qtj6WPb9qQIgWyP4I4J4OCXv/+dvANKCGltowCn98XXdrhh + 8npf0YeONnBGio4bJht6H6VTCBx/hQgdB1Qe8Ja++5igchXNoTMuzB399hMvTKV+/FOHDZ0PeSL6 + hmJXu7eF7ORfy2/1tuZ4XJzDMASiTvW9xAoU6vvPJkkK6c1LFuA0RA6DGyA4kzWsa/2laX+q8xYe + EMGDAExsPeU1ltVxfEPrDAQQv6kXHaoZdHA5t4NZwjeXth+RP/mkcIEnFJbXT70Z7uuqYg/5OOYn + Nre3hDRsAY0TqqKkQN+tFVnZ4i9pcZ148BZZ/5gDh0ZQVWja3hIWcYuMJUyQc4ktSLDSZeqlMXlC + tmrTWRTxFtkRz8IVbD0iP5GtypwqzeEy4RgoaPfW42ZKhTyjCSur6RQH1+d0gnK3nNg77f1jUoDW + ol79Cgmbiqed8vKAQUjItie0RwrO6L8ebaNitXWIG69P6BEh1M/TOIZVnV9EPpDW/fW1MzgxT8+P + D35+cXZ4+vSZ9fZkcNx9vOybRVdV36OjZ788O31+evzm6OD8xcnv9rOfX5sG79usJ6kWFOeexn6a + YOUbBi+ixE8XqnjGEa0G6LAUCCEP5OKiIgA7VDUXwVfBWC+/XVi61ZQfSQNC/J8PQAK2K6fYvVsE + HoEMNO6zUAUt4mPnHVJ7KSq318gAt4pmoLKhsMJcIPnZZJjDltyvkuO1tZJBccHl1P9Euv0Pfb4L + D7AzkLS64/b2J4QMjTfILZrNZBbH71Ha4KmyJ/eXUEk8H6k0Q79xGEaxvw196olFwXaz9I/+bG63 + H/31Fxff7Ue851J+//Y3HMaNw24fv2LiZ+n5fxrT+T8T/YhcRkYAAA== + headers: + Server: [nginx] + Content-Type: [text/html] + Vary: [Accept-Encoding] + Cache-Control: [no-cache] + must-revalidate: ['s-maxage=3600'] + Expires: ['Fri 15 Oct 2004 12:00:00 GMT'] + Server-Name: ['dalmozwww01.dal.moz.com'] + Content-Encoding: [gzip] + Content-Length: ['5683'] + Accept-Ranges: [bytes] + Date: ['Sat, 11 Jan 2014 18:45:11 GMT'] + X-Varnish: [918768771 918700396] + Age: ['3479'] + Via: ['1.1 varnish'] + Connection: [keep-alive] + status: {code: 200, message: OK} diff --git a/tools/vcrpy/tests/integration/__init__.py b/tools/vcrpy/tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tools/vcrpy/tests/integration/aiohttp_utils.py b/tools/vcrpy/tests/integration/aiohttp_utils.py new file mode 100644 index 000000000000..b2769733e6c5 --- /dev/null +++ b/tools/vcrpy/tests/integration/aiohttp_utils.py @@ -0,0 +1,43 @@ +# flake8: noqa +import asyncio + +import aiohttp +from aiohttp.test_utils import TestClient + + +async def aiohttp_request(loop, method, url, output="text", encoding="utf-8", content_type=None, **kwargs): + session = aiohttp.ClientSession(loop=loop) + response_ctx = session.request(method, url, **kwargs) + + response = await response_ctx.__aenter__() + if output == "text": + content = await response.text() + elif output == "json": + content_type = content_type or "application/json" + content = await response.json(encoding=encoding, content_type=content_type) + elif output == "raw": + content = await response.read() + elif output == "stream": + content = await response.content.read() + + response_ctx._resp.close() + await session.close() + + return response, content + + +def aiohttp_app(): + async def hello(request): + return aiohttp.web.Response(text="hello") + + async def json(request): + return aiohttp.web.json_response({}) + + async def json_empty_body(request): + return aiohttp.web.json_response() + + app = aiohttp.web.Application() + app.router.add_get("/", hello) + app.router.add_get("/json", json) + app.router.add_get("/json/empty", json_empty_body) + return app diff --git a/tools/vcrpy/tests/integration/test_aiohttp.py b/tools/vcrpy/tests/integration/test_aiohttp.py new file mode 100644 index 000000000000..6508c4871f51 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_aiohttp.py @@ -0,0 +1,304 @@ +import contextlib +import logging + +import pytest + +asyncio = pytest.importorskip("asyncio") +aiohttp = pytest.importorskip("aiohttp") + +import vcr # noqa: E402 +from .aiohttp_utils import aiohttp_app, aiohttp_request # noqa: E402 + + +def run_in_loop(fn): + with contextlib.closing(asyncio.new_event_loop()) as loop: + asyncio.set_event_loop(loop) + task = loop.create_task(fn(loop)) + return loop.run_until_complete(task) + + +def request(method, url, output="text", **kwargs): + def run(loop): + return aiohttp_request(loop, method, url, output=output, **kwargs) + + return run_in_loop(run) + + +def get(url, output="text", **kwargs): + return request("GET", url, output=output, **kwargs) + + +def post(url, output="text", **kwargs): + return request("POST", url, output="text", **kwargs) + + +@pytest.fixture(params=["https", "http"]) +def scheme(request): + """Fixture that returns both http and https.""" + return request.param + + +def test_status(tmpdir, scheme): + url = scheme + "://httpbin.org" + with vcr.use_cassette(str(tmpdir.join("status.yaml"))): + response, _ = get(url) + + with vcr.use_cassette(str(tmpdir.join("status.yaml"))) as cassette: + cassette_response, _ = get(url) + assert cassette_response.status == response.status + assert cassette.play_count == 1 + + +@pytest.mark.parametrize("auth", [None, aiohttp.BasicAuth("vcrpy", "test")]) +def test_headers(tmpdir, scheme, auth): + url = scheme + "://httpbin.org" + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + response, _ = get(url, auth=auth) + + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))) as cassette: + if auth is not None: + request = cassette.requests[0] + assert "AUTHORIZATION" in request.headers + cassette_response, _ = get(url, auth=auth) + assert dict(cassette_response.headers) == dict(response.headers) + assert cassette.play_count == 1 + assert "istr" not in cassette.data[0] + assert "yarl.URL" not in cassette.data[0] + + +def test_case_insensitive_headers(tmpdir, scheme): + url = scheme + "://httpbin.org" + with vcr.use_cassette(str(tmpdir.join("whatever.yaml"))): + _, _ = get(url) + + with vcr.use_cassette(str(tmpdir.join("whatever.yaml"))) as cassette: + cassette_response, _ = get(url) + assert "Content-Type" in cassette_response.headers + assert "content-type" in cassette_response.headers + assert cassette.play_count == 1 + + +def test_text(tmpdir, scheme): + url = scheme + "://httpbin.org" + with vcr.use_cassette(str(tmpdir.join("text.yaml"))): + _, response_text = get(url) + + with vcr.use_cassette(str(tmpdir.join("text.yaml"))) as cassette: + _, cassette_response_text = get(url) + assert cassette_response_text == response_text + assert cassette.play_count == 1 + + +def test_json(tmpdir, scheme): + url = scheme + "://httpbin.org/get" + headers = {"Content-Type": "application/json"} + + with vcr.use_cassette(str(tmpdir.join("json.yaml"))): + _, response_json = get(url, output="json", headers=headers) + + with vcr.use_cassette(str(tmpdir.join("json.yaml"))) as cassette: + _, cassette_response_json = get(url, output="json", headers=headers) + assert cassette_response_json == response_json + assert cassette.play_count == 1 + + +def test_binary(tmpdir, scheme): + url = scheme + "://httpbin.org/image/png" + with vcr.use_cassette(str(tmpdir.join("binary.yaml"))): + _, response_binary = get(url, output="raw") + + with vcr.use_cassette(str(tmpdir.join("binary.yaml"))) as cassette: + _, cassette_response_binary = get(url, output="raw") + assert cassette_response_binary == response_binary + assert cassette.play_count == 1 + + +def test_stream(tmpdir, scheme): + url = scheme + "://httpbin.org/get" + + with vcr.use_cassette(str(tmpdir.join("stream.yaml"))): + resp, body = get(url, output="raw") # Do not use stream here, as the stream is exhausted by vcr + + with vcr.use_cassette(str(tmpdir.join("stream.yaml"))) as cassette: + cassette_resp, cassette_body = get(url, output="stream") + assert cassette_body == body + assert cassette.play_count == 1 + + +@pytest.mark.parametrize("body", ["data", "json"]) +def test_post(tmpdir, scheme, body, caplog): + caplog.set_level(logging.INFO) + data = {"key1": "value1", "key2": "value2"} + url = scheme + "://httpbin.org/post" + with vcr.use_cassette(str(tmpdir.join("post.yaml"))): + _, response_json = post(url, **{body: data}) + + with vcr.use_cassette(str(tmpdir.join("post.yaml"))) as cassette: + request = cassette.requests[0] + assert request.body == data + _, cassette_response_json = post(url, **{body: data}) + assert cassette_response_json == response_json + assert cassette.play_count == 1 + + assert next( + ( + log + for log in caplog.records + if log.getMessage() == " not in cassette, sending to real server".format(url) + ), + None, + ), "Log message not found." + + +def test_params(tmpdir, scheme): + url = scheme + "://httpbin.org/get" + headers = {"Content-Type": "application/json"} + params = {"a": 1, "b": False, "c": "c"} + + with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: + _, response_json = get(url, output="json", params=params, headers=headers) + + with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: + _, cassette_response_json = get(url, output="json", params=params, headers=headers) + assert cassette_response_json == response_json + assert cassette.play_count == 1 + + +def test_params_same_url_distinct_params(tmpdir, scheme): + url = scheme + "://httpbin.org/get" + headers = {"Content-Type": "application/json"} + params = {"a": 1, "b": False, "c": "c"} + + with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: + _, response_json = get(url, output="json", params=params, headers=headers) + + with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: + _, cassette_response_json = get(url, output="json", params=params, headers=headers) + assert cassette_response_json == response_json + assert cassette.play_count == 1 + + other_params = {"other": "params"} + with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: + response, cassette_response_text = get(url, output="text", params=other_params) + assert "No match for the request" in cassette_response_text + assert response.status == 599 + + +def test_params_on_url(tmpdir, scheme): + url = scheme + "://httpbin.org/get?a=1&b=foo" + headers = {"Content-Type": "application/json"} + + with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: + _, response_json = get(url, output="json", headers=headers) + request = cassette.requests[0] + assert request.url == url + + with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: + _, cassette_response_json = get(url, output="json", headers=headers) + request = cassette.requests[0] + assert request.url == url + assert cassette_response_json == response_json + assert cassette.play_count == 1 + + +def test_aiohttp_test_client(aiohttp_client, tmpdir): + loop = asyncio.get_event_loop() + app = aiohttp_app() + url = "/" + client = loop.run_until_complete(aiohttp_client(app)) + + with vcr.use_cassette(str(tmpdir.join("get.yaml"))): + response = loop.run_until_complete(client.get(url)) + + assert response.status == 200 + response_text = loop.run_until_complete(response.text()) + assert response_text == "hello" + response_text = loop.run_until_complete(response.text(errors="replace")) + assert response_text == "hello" + + with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: + response = loop.run_until_complete(client.get(url)) + + request = cassette.requests[0] + assert request.url == str(client.make_url(url)) + response_text = loop.run_until_complete(response.text()) + assert response_text == "hello" + assert cassette.play_count == 1 + + +def test_aiohttp_test_client_json(aiohttp_client, tmpdir): + loop = asyncio.get_event_loop() + app = aiohttp_app() + url = "/json/empty" + client = loop.run_until_complete(aiohttp_client(app)) + + with vcr.use_cassette(str(tmpdir.join("get.yaml"))): + response = loop.run_until_complete(client.get(url)) + + assert response.status == 200 + response_json = loop.run_until_complete(response.json()) + assert response_json is None + + with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: + response = loop.run_until_complete(client.get(url)) + + request = cassette.requests[0] + assert request.url == str(client.make_url(url)) + response_json = loop.run_until_complete(response.json()) + assert response_json is None + assert cassette.play_count == 1 + + +def test_redirect(aiohttp_client, tmpdir): + url = "https://httpbin.org/redirect/2" + + with vcr.use_cassette(str(tmpdir.join("redirect.yaml"))): + response, _ = get(url) + + with vcr.use_cassette(str(tmpdir.join("redirect.yaml"))) as cassette: + cassette_response, _ = get(url) + + assert cassette_response.status == response.status + assert len(cassette_response.history) == len(response.history) + assert len(cassette) == 3 + assert cassette.play_count == 3 + + # Assert that the real response and the cassette response have a similar + # looking request_info. + assert cassette_response.request_info.url == response.request_info.url + assert cassette_response.request_info.method == response.request_info.method + assert {k: v for k, v in cassette_response.request_info.headers.items()} == { + k: v for k, v in response.request_info.headers.items() + } + assert cassette_response.request_info.real_url == response.request_info.real_url + + +def test_double_requests(tmpdir): + """We should capture, record, and replay all requests and response chains, + even if there are duplicate ones. + + We should replay in the order we saw them. + """ + url = "https://httpbin.org/get" + + with vcr.use_cassette(str(tmpdir.join("text.yaml"))): + _, response_text1 = get(url, output="text") + _, response_text2 = get(url, output="text") + + with vcr.use_cassette(str(tmpdir.join("text.yaml"))) as cassette: + resp, cassette_response_text = get(url, output="text") + assert resp.status == 200 + assert cassette_response_text == response_text1 + + # We made only one request, so we should only play 1 recording. + assert cassette.play_count == 1 + + # Now make the second test to url + resp, cassette_response_text = get(url, output="text") + + assert resp.status == 200 + + assert cassette_response_text == response_text2 + + # Now that we made both requests, we should have played both. + assert cassette.play_count == 2 diff --git a/tools/vcrpy/tests/integration/test_basic.py b/tools/vcrpy/tests/integration/test_basic.py new file mode 100644 index 000000000000..bdcc4ed7928e --- /dev/null +++ b/tools/vcrpy/tests/integration/test_basic.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +"""Basic tests for cassettes""" + +# External imports +import os +from six.moves.urllib.request import urlopen + +# Internal imports +import vcr + + +def test_nonexistent_directory(tmpdir, httpbin): + """If we load a cassette in a nonexistent directory, it can save ok""" + # Check to make sure directory doesnt exist + assert not os.path.exists(str(tmpdir.join("nonexistent"))) + + # Run VCR to create dir and cassette file + with vcr.use_cassette(str(tmpdir.join("nonexistent", "cassette.yml"))): + urlopen(httpbin.url).read() + + # This should have made the file and the directory + assert os.path.exists(str(tmpdir.join("nonexistent", "cassette.yml"))) + + +def test_unpatch(tmpdir, httpbin): + """Ensure that our cassette gets unpatched when we're done""" + with vcr.use_cassette(str(tmpdir.join("unpatch.yaml"))) as cass: + urlopen(httpbin.url).read() + + # Make the same request, and assert that we haven't served any more + # requests out of cache + urlopen(httpbin.url).read() + assert cass.play_count == 0 + + +def test_basic_json_use(tmpdir, httpbin): + """ + Ensure you can load a json serialized cassette + """ + test_fixture = str(tmpdir.join("synopsis.json")) + with vcr.use_cassette(test_fixture, serializer="json"): + response = urlopen(httpbin.url).read() + assert b"difficult sometimes" in response + + +def test_patched_content(tmpdir, httpbin): + """ + Ensure that what you pull from a cassette is what came from the + request + """ + with vcr.use_cassette(str(tmpdir.join("synopsis.yaml"))) as cass: + response = urlopen(httpbin.url).read() + assert cass.play_count == 0 + + with vcr.use_cassette(str(tmpdir.join("synopsis.yaml"))) as cass: + response2 = urlopen(httpbin.url).read() + assert cass.play_count == 1 + cass._save(force=True) + + with vcr.use_cassette(str(tmpdir.join("synopsis.yaml"))) as cass: + response3 = urlopen(httpbin.url).read() + assert cass.play_count == 1 + + assert response == response2 + assert response2 == response3 + + +def test_patched_content_json(tmpdir, httpbin): + """ + Ensure that what you pull from a json cassette is what came from the + request + """ + + testfile = str(tmpdir.join("synopsis.json")) + + with vcr.use_cassette(testfile) as cass: + response = urlopen(httpbin.url).read() + assert cass.play_count == 0 + + with vcr.use_cassette(testfile) as cass: + response2 = urlopen(httpbin.url).read() + assert cass.play_count == 1 + cass._save(force=True) + + with vcr.use_cassette(testfile) as cass: + response3 = urlopen(httpbin.url).read() + assert cass.play_count == 1 + + assert response == response2 + assert response2 == response3 diff --git a/tools/vcrpy/tests/integration/test_boto.py b/tools/vcrpy/tests/integration/test_boto.py new file mode 100644 index 000000000000..4087fc974984 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_boto.py @@ -0,0 +1,84 @@ +import pytest + +boto = pytest.importorskip("boto") + +import boto # NOQA +import boto.iam # NOQA +from boto.s3.connection import S3Connection # NOQA +from boto.s3.key import Key # NOQA +import vcr # NOQA + +try: # NOQA + from ConfigParser import DuplicateSectionError # NOQA +except ImportError: # NOQA + # python3 + from configparser import DuplicateSectionError # NOQA + + +def test_boto_stubs(tmpdir): + with vcr.use_cassette(str(tmpdir.join("boto-stubs.yml"))): + # Perform the imports within the patched context so that + # CertValidatingHTTPSConnection refers to the patched version. + from boto.https_connection import CertValidatingHTTPSConnection + from vcr.stubs.boto_stubs import VCRCertValidatingHTTPSConnection + + # Prove that the class was patched by the stub and that we can instantiate it. + assert issubclass(CertValidatingHTTPSConnection, VCRCertValidatingHTTPSConnection) + CertValidatingHTTPSConnection("hostname.does.not.matter") + + +def test_boto_without_vcr(): + s3_conn = S3Connection() + s3_bucket = s3_conn.get_bucket("boto-demo-1394171994") # a bucket you can access + k = Key(s3_bucket) + k.key = "test.txt" + k.set_contents_from_string("hello world i am a string") + + +def test_boto_medium_difficulty(tmpdir): + s3_conn = S3Connection() + s3_bucket = s3_conn.get_bucket("boto-demo-1394171994") # a bucket you can access + with vcr.use_cassette(str(tmpdir.join("boto-medium.yml"))): + k = Key(s3_bucket) + k.key = "test.txt" + k.set_contents_from_string("hello world i am a string") + + with vcr.use_cassette(str(tmpdir.join("boto-medium.yml"))): + k = Key(s3_bucket) + k.key = "test.txt" + k.set_contents_from_string("hello world i am a string") + + +def test_boto_hardcore_mode(tmpdir): + with vcr.use_cassette(str(tmpdir.join("boto-hardcore.yml"))): + s3_conn = S3Connection() + s3_bucket = s3_conn.get_bucket("boto-demo-1394171994") # a bucket you can access + k = Key(s3_bucket) + k.key = "test.txt" + k.set_contents_from_string("hello world i am a string") + + with vcr.use_cassette(str(tmpdir.join("boto-hardcore.yml"))): + s3_conn = S3Connection() + s3_bucket = s3_conn.get_bucket("boto-demo-1394171994") # a bucket you can access + k = Key(s3_bucket) + k.key = "test.txt" + k.set_contents_from_string("hello world i am a string") + + +def test_boto_iam(tmpdir): + try: + boto.config.add_section("Boto") + except DuplicateSectionError: + pass + # Ensure that boto uses HTTPS + boto.config.set("Boto", "is_secure", "true") + # Ensure that boto uses CertValidatingHTTPSConnection + boto.config.set("Boto", "https_validate_certificates", "true") + + with vcr.use_cassette(str(tmpdir.join("boto-iam.yml"))): + iam_conn = boto.iam.connect_to_region("universal") + iam_conn.get_all_users() + + with vcr.use_cassette(str(tmpdir.join("boto-iam.yml"))): + iam_conn = boto.iam.connect_to_region("universal") + iam_conn.get_all_users() diff --git a/tools/vcrpy/tests/integration/test_boto3.py b/tools/vcrpy/tests/integration/test_boto3.py new file mode 100644 index 000000000000..ee93ffb4a994 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_boto3.py @@ -0,0 +1,122 @@ +import pytest +import os + +boto3 = pytest.importorskip("boto3") + +import boto3 # NOQA +import botocore # NOQA +import vcr # NOQA + +try: + from botocore import awsrequest # NOQA + + botocore_awsrequest = True +except ImportError: + botocore_awsrequest = False + +# skip tests if boto does not use vendored requests anymore +# https://github.com/boto/botocore/pull/1495 +boto3_skip_vendored_requests = pytest.mark.skipif( + botocore_awsrequest, + reason="botocore version {ver} does not use vendored requests anymore.".format(ver=botocore.__version__), +) + +boto3_skip_awsrequest = pytest.mark.skipif( + not botocore_awsrequest, + reason="botocore version {ver} still uses vendored requests.".format(ver=botocore.__version__), +) + +IAM_USER_NAME = "vcrpy" + + +@pytest.fixture +def iam_client(): + def _iam_client(boto3_session=None): + if boto3_session is None: + boto3_session = boto3.Session( + aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", "default"), + aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", "default"), + aws_session_token=None, + region_name=os.environ.get("AWS_DEFAULT_REGION", "default"), + ) + return boto3_session.client("iam") + + return _iam_client + + +@pytest.fixture +def get_user(iam_client): + def _get_user(client=None, user_name=IAM_USER_NAME): + if client is None: + # Default client set with fixture `iam_client` + client = iam_client() + return client.get_user(UserName=user_name) + + return _get_user + + +@boto3_skip_vendored_requests +def test_boto_vendored_stubs(tmpdir): + with vcr.use_cassette(str(tmpdir.join("boto3-stubs.yml"))): + # Perform the imports within the patched context so that + # HTTPConnection, VerifiedHTTPSConnection refers to the patched version. + from botocore.vendored.requests.packages.urllib3.connectionpool import ( + HTTPConnection, + VerifiedHTTPSConnection, + ) + from vcr.stubs.boto3_stubs import VCRRequestsHTTPConnection, VCRRequestsHTTPSConnection + + # Prove that the class was patched by the stub and that we can instantiate it. + assert issubclass(HTTPConnection, VCRRequestsHTTPConnection) + assert issubclass(VerifiedHTTPSConnection, VCRRequestsHTTPSConnection) + HTTPConnection("hostname.does.not.matter") + VerifiedHTTPSConnection("hostname.does.not.matter") + + +@pytest.mark.skipif( + os.environ.get("TRAVIS_PULL_REQUEST") != "false", + reason="Encrypted Environment Variables from Travis Repository Settings" + " are disabled on PRs from forks. " + "https://docs.travis-ci.com/user/pull-requests/#pull-requests-and-security-restrictions", +) +def test_boto_medium_difficulty(tmpdir, get_user): + + with vcr.use_cassette(str(tmpdir.join("boto3-medium.yml"))): + response = get_user() + assert response["User"]["UserName"] == IAM_USER_NAME + + with vcr.use_cassette(str(tmpdir.join("boto3-medium.yml"))) as cass: + response = get_user() + assert response["User"]["UserName"] == IAM_USER_NAME + assert cass.all_played + + +@pytest.mark.skipif( + os.environ.get("TRAVIS_PULL_REQUEST") != "false", + reason="Encrypted Environment Variables from Travis Repository Settings" + " are disabled on PRs from forks. " + "https://docs.travis-ci.com/user/pull-requests/#pull-requests-and-security-restrictions", +) +def test_boto_hardcore_mode(tmpdir, iam_client, get_user): + with vcr.use_cassette(str(tmpdir.join("boto3-hardcore.yml"))): + ses = boto3.Session( + aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"), + aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"), + region_name=os.environ.get("AWS_DEFAULT_REGION"), + ) + client = iam_client(ses) + response = get_user(client=client) + assert response["User"]["UserName"] == IAM_USER_NAME + + with vcr.use_cassette(str(tmpdir.join("boto3-hardcore.yml"))) as cass: + ses = boto3.Session( + aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"), + aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"), + aws_session_token=None, + region_name=os.environ.get("AWS_DEFAULT_REGION"), + ) + + client = iam_client(ses) + response = get_user(client=client) + assert response["User"]["UserName"] == IAM_USER_NAME + assert cass.all_played diff --git a/tools/vcrpy/tests/integration/test_config.py b/tools/vcrpy/tests/integration/test_config.py new file mode 100644 index 000000000000..e0776bc87fcd --- /dev/null +++ b/tools/vcrpy/tests/integration/test_config.py @@ -0,0 +1,58 @@ +import os +import json +import pytest +import vcr +from six.moves.urllib.request import urlopen + + +def test_set_serializer_default_config(tmpdir, httpbin): + my_vcr = vcr.VCR(serializer="json") + + with my_vcr.use_cassette(str(tmpdir.join("test.json"))): + assert my_vcr.serializer == "json" + urlopen(httpbin.url + "/get") + + with open(str(tmpdir.join("test.json"))) as f: + assert json.loads(f.read()) + + +def test_default_set_cassette_library_dir(tmpdir, httpbin): + my_vcr = vcr.VCR(cassette_library_dir=str(tmpdir.join("subdir"))) + + with my_vcr.use_cassette("test.json"): + urlopen(httpbin.url + "/get") + + assert os.path.exists(str(tmpdir.join("subdir").join("test.json"))) + + +def test_override_set_cassette_library_dir(tmpdir, httpbin): + my_vcr = vcr.VCR(cassette_library_dir=str(tmpdir.join("subdir"))) + + cld = str(tmpdir.join("subdir2")) + + with my_vcr.use_cassette("test.json", cassette_library_dir=cld): + urlopen(httpbin.url + "/get") + + assert os.path.exists(str(tmpdir.join("subdir2").join("test.json"))) + assert not os.path.exists(str(tmpdir.join("subdir").join("test.json"))) + + +def test_override_match_on(tmpdir, httpbin): + my_vcr = vcr.VCR(match_on=["method"]) + + with my_vcr.use_cassette(str(tmpdir.join("test.json"))): + urlopen(httpbin.url) + + with my_vcr.use_cassette(str(tmpdir.join("test.json"))) as cass: + urlopen(httpbin.url + "/get") + + assert len(cass) == 1 + assert cass.play_count == 1 + + +def test_missing_matcher(): + my_vcr = vcr.VCR() + my_vcr.register_matcher("awesome", object) + with pytest.raises(KeyError): + with my_vcr.use_cassette("test.yaml", match_on=["notawesome"]): + pass diff --git a/tools/vcrpy/tests/integration/test_disksaver.py b/tools/vcrpy/tests/integration/test_disksaver.py new file mode 100644 index 000000000000..c3e2e8c3bd69 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_disksaver.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +"""Basic tests about save behavior""" + +# External imports +import os +import time +from six.moves.urllib.request import urlopen + +# Internal imports +import vcr + + +def test_disk_saver_nowrite(tmpdir, httpbin): + """ + Ensure that when you close a cassette without changing it it doesn't + rewrite the file + """ + fname = str(tmpdir.join("synopsis.yaml")) + with vcr.use_cassette(fname) as cass: + urlopen(httpbin.url).read() + assert cass.play_count == 0 + last_mod = os.path.getmtime(fname) + + with vcr.use_cassette(fname) as cass: + urlopen(httpbin.url).read() + assert cass.play_count == 1 + assert cass.dirty is False + last_mod2 = os.path.getmtime(fname) + + assert last_mod == last_mod2 + + +def test_disk_saver_write(tmpdir, httpbin): + """ + Ensure that when you close a cassette after changing it it does + rewrite the file + """ + fname = str(tmpdir.join("synopsis.yaml")) + with vcr.use_cassette(fname) as cass: + urlopen(httpbin.url).read() + assert cass.play_count == 0 + last_mod = os.path.getmtime(fname) + + # Make sure at least 1 second passes, otherwise sometimes + # the mtime doesn't change + time.sleep(1) + + with vcr.use_cassette(fname, record_mode="any") as cass: + urlopen(httpbin.url).read() + urlopen(httpbin.url + "/get").read() + assert cass.play_count == 1 + assert cass.dirty + last_mod2 = os.path.getmtime(fname) + + assert last_mod != last_mod2 diff --git a/tools/vcrpy/tests/integration/test_filter.py b/tools/vcrpy/tests/integration/test_filter.py new file mode 100644 index 000000000000..6ec859460aa0 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_filter.py @@ -0,0 +1,130 @@ +import base64 +import pytest +from six.moves.urllib.request import urlopen, Request +from six.moves.urllib.parse import urlencode +from six.moves.urllib.error import HTTPError +import vcr +import json +from assertions import assert_cassette_has_one_response, assert_is_json + + +def _request_with_auth(url, username, password): + request = Request(url) + base64string = base64.b64encode(username.encode("ascii") + b":" + password.encode("ascii")) + request.add_header(b"Authorization", b"Basic " + base64string) + return urlopen(request) + + +def _find_header(cassette, header): + return any(header in request.headers for request in cassette.requests) + + +def test_filter_basic_auth(tmpdir, httpbin): + url = httpbin.url + "/basic-auth/user/passwd" + cass_file = str(tmpdir.join("basic_auth_filter.yaml")) + my_vcr = vcr.VCR(match_on=["uri", "method", "headers"]) + # 2 requests, one with auth failure and one with auth success + with my_vcr.use_cassette(cass_file, filter_headers=["authorization"]): + with pytest.raises(HTTPError): + resp = _request_with_auth(url, "user", "wrongpasswd") + assert resp.getcode() == 401 + resp = _request_with_auth(url, "user", "passwd") + assert resp.getcode() == 200 + # make same 2 requests, this time both served from cassette. + with my_vcr.use_cassette(cass_file, filter_headers=["authorization"]) as cass: + with pytest.raises(HTTPError): + resp = _request_with_auth(url, "user", "wrongpasswd") + assert resp.getcode() == 401 + resp = _request_with_auth(url, "user", "passwd") + assert resp.getcode() == 200 + # authorization header should not have been recorded + assert not _find_header(cass, "authorization") + assert len(cass) == 2 + + +def test_filter_querystring(tmpdir, httpbin): + url = httpbin.url + "/?foo=bar" + cass_file = str(tmpdir.join("filter_qs.yaml")) + with vcr.use_cassette(cass_file, filter_query_parameters=["foo"]): + urlopen(url) + with vcr.use_cassette(cass_file, filter_query_parameters=["foo"]) as cass: + urlopen(url) + assert "foo" not in cass.requests[0].url + + +def test_filter_post_data(tmpdir, httpbin): + url = httpbin.url + "/post" + data = urlencode({"id": "secret", "foo": "bar"}).encode("utf-8") + cass_file = str(tmpdir.join("filter_pd.yaml")) + with vcr.use_cassette(cass_file, filter_post_data_parameters=["id"]): + urlopen(url, data) + with vcr.use_cassette(cass_file, filter_post_data_parameters=["id"]) as cass: + assert b"id=secret" not in cass.requests[0].body + + +def test_filter_json_post_data(tmpdir, httpbin): + data = json.dumps({"id": "secret", "foo": "bar"}).encode("utf-8") + request = Request(httpbin.url + "/post", data=data) + request.add_header("Content-Type", "application/json") + + cass_file = str(tmpdir.join("filter_jpd.yaml")) + with vcr.use_cassette(cass_file, filter_post_data_parameters=["id"]): + urlopen(request) + with vcr.use_cassette(cass_file, filter_post_data_parameters=["id"]) as cass: + assert b'"id": "secret"' not in cass.requests[0].body + + +def test_filter_callback(tmpdir, httpbin): + url = httpbin.url + "/get" + cass_file = str(tmpdir.join("basic_auth_filter.yaml")) + + def before_record_cb(request): + if request.path != "/get": + return request + + # Test the legacy keyword. + my_vcr = vcr.VCR(before_record=before_record_cb) + with my_vcr.use_cassette(cass_file, filter_headers=["authorization"]) as cass: + urlopen(url) + assert len(cass) == 0 + + my_vcr = vcr.VCR(before_record_request=before_record_cb) + with my_vcr.use_cassette(cass_file, filter_headers=["authorization"]) as cass: + urlopen(url) + assert len(cass) == 0 + + +def test_decompress_gzip(tmpdir, httpbin): + url = httpbin.url + "/gzip" + request = Request(url, headers={"Accept-Encoding": ["gzip, deflate"]}) + cass_file = str(tmpdir.join("gzip_response.yaml")) + with vcr.use_cassette(cass_file, decode_compressed_response=True): + urlopen(request) + with vcr.use_cassette(cass_file) as cass: + decoded_response = urlopen(url).read() + assert_cassette_has_one_response(cass) + assert_is_json(decoded_response) + + +def test_decompress_deflate(tmpdir, httpbin): + url = httpbin.url + "/deflate" + request = Request(url, headers={"Accept-Encoding": ["gzip, deflate"]}) + cass_file = str(tmpdir.join("deflate_response.yaml")) + with vcr.use_cassette(cass_file, decode_compressed_response=True): + urlopen(request) + with vcr.use_cassette(cass_file) as cass: + decoded_response = urlopen(url).read() + assert_cassette_has_one_response(cass) + assert_is_json(decoded_response) + + +def test_decompress_regular(tmpdir, httpbin): + """Test that it doesn't try to decompress content that isn't compressed""" + url = httpbin.url + "/get" + cass_file = str(tmpdir.join("noncompressed_response.yaml")) + with vcr.use_cassette(cass_file, decode_compressed_response=True): + urlopen(url) + with vcr.use_cassette(cass_file) as cass: + resp = urlopen(url).read() + assert_cassette_has_one_response(cass) + assert_is_json(resp) diff --git a/tools/vcrpy/tests/integration/test_http b/tools/vcrpy/tests/integration/test_http new file mode 100644 index 000000000000..522363be3a54 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_http @@ -0,0 +1,22 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: https://httpbin.org/get?ham=spam + response: + body: {string: "{\n \"args\": {\n \"ham\": \"spam\"\n }, \n \"headers\"\ + : {\n \"Accept\": \"*/*\", \n \"Accept-Encoding\": \"gzip, deflate\"\ + , \n \"Connection\": \"close\", \n \"Host\": \"httpbin.org\", \n \ + \ \"User-Agent\": \"Python/3.5 aiohttp/2.0.1\"\n }, \n \"origin\": \"213.86.221.35\"\ + , \n \"url\": \"https://httpbin.org/get?ham=spam\"\n}\n"} + headers: {Access-Control-Allow-Credentials: 'true', Access-Control-Allow-Origin: '*', + Connection: keep-alive, Content-Length: '299', Content-Type: application/json, + Date: 'Wed, 22 Mar 2017 20:08:29 GMT', Server: gunicorn/19.7.1, Via: 1.1 vegur} + status: {code: 200, message: OK} + url: !!python/object/new:yarl.URL + state: !!python/tuple + - !!python/object/new:urllib.parse.SplitResult [https, httpbin.org, /get, ham=spam, + ''] + - false +version: 1 diff --git a/tools/vcrpy/tests/integration/test_httplib2.py b/tools/vcrpy/tests/integration/test_httplib2.py new file mode 100644 index 000000000000..7d05c90db145 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_httplib2.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +"""Integration tests with httplib2""" + +import sys + +from six.moves.urllib_parse import urlencode +import pytest +import pytest_httpbin.certs + +import vcr + +from assertions import assert_cassette_has_one_response + +httplib2 = pytest.importorskip("httplib2") + + +def http(): + """ + Returns an httplib2 HTTP instance + with the certificate replaced by the httpbin one. + """ + kwargs = {"ca_certs": pytest_httpbin.certs.where()} + if sys.version_info[:2] in [(2, 7), (3, 7)]: + kwargs["disable_ssl_certificate_validation"] = True + return httplib2.Http(**kwargs) + + +def test_response_code(tmpdir, httpbin_both): + """Ensure we can read a response code from a fetch""" + url = httpbin_both.url + with vcr.use_cassette(str(tmpdir.join("atts.yaml"))): + resp, _ = http().request(url) + code = resp.status + + with vcr.use_cassette(str(tmpdir.join("atts.yaml"))): + resp, _ = http().request(url) + assert code == resp.status + + +def test_random_body(httpbin_both, tmpdir): + """Ensure we can read the content, and that it's served from cache""" + url = httpbin_both.url + "/bytes/1024" + with vcr.use_cassette(str(tmpdir.join("body.yaml"))): + _, content = http().request(url) + body = content + + with vcr.use_cassette(str(tmpdir.join("body.yaml"))): + _, content = http().request(url) + assert body == content + + +def test_response_headers(tmpdir, httpbin_both): + """Ensure we can get information from the response""" + url = httpbin_both.url + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + resp, _ = http().request(url) + headers = resp.items() + + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + resp, _ = http().request(url) + assert set(headers) == set(resp.items()) + + +def test_effective_url(tmpdir, httpbin_both): + """Ensure that the effective_url is captured""" + url = httpbin_both.url + "/redirect-to?url=/html" + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + resp, _ = http().request(url) + effective_url = resp["content-location"] + assert effective_url == httpbin_both + "/html" + + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + resp, _ = http().request(url) + assert effective_url == resp["content-location"] + + +def test_multiple_requests(tmpdir, httpbin_both): + """Ensure that we can cache multiple requests""" + urls = [httpbin_both.url, httpbin_both.url, httpbin_both.url + "/get", httpbin_both.url + "/bytes/1024"] + with vcr.use_cassette(str(tmpdir.join("multiple.yaml"))) as cass: + [http().request(url) for url in urls] + assert len(cass) == len(urls) + + +def test_get_data(tmpdir, httpbin_both): + """Ensure that it works with query data""" + data = urlencode({"some": 1, "data": "here"}) + url = httpbin_both.url + "/get?" + data + with vcr.use_cassette(str(tmpdir.join("get_data.yaml"))): + _, res1 = http().request(url) + + with vcr.use_cassette(str(tmpdir.join("get_data.yaml"))): + _, res2 = http().request(url) + + assert res1 == res2 + + +def test_post_data(tmpdir, httpbin_both): + """Ensure that it works when posting data""" + data = urlencode({"some": 1, "data": "here"}) + url = httpbin_both.url + "/post" + with vcr.use_cassette(str(tmpdir.join("post_data.yaml"))): + _, res1 = http().request(url, "POST", data) + + with vcr.use_cassette(str(tmpdir.join("post_data.yaml"))) as cass: + _, res2 = http().request(url, "POST", data) + + assert res1 == res2 + assert_cassette_has_one_response(cass) + + +def test_post_unicode_data(tmpdir, httpbin_both): + """Ensure that it works when posting unicode data""" + data = urlencode({"snowman": u"☃".encode("utf-8")}) + url = httpbin_both.url + "/post" + with vcr.use_cassette(str(tmpdir.join("post_data.yaml"))): + _, res1 = http().request(url, "POST", data) + + with vcr.use_cassette(str(tmpdir.join("post_data.yaml"))) as cass: + _, res2 = http().request(url, "POST", data) + + assert res1 == res2 + assert_cassette_has_one_response(cass) + + +def test_cross_scheme(tmpdir, httpbin, httpbin_secure): + """Ensure that requests between schemes are treated separately""" + # First fetch a url under https, and then again under https and then + # ensure that we haven't served anything out of cache, and we have two + # requests / response pairs in the cassette + with vcr.use_cassette(str(tmpdir.join("cross_scheme.yaml"))) as cass: + http().request(httpbin_secure.url) + http().request(httpbin.url) + assert len(cass) == 2 + assert cass.play_count == 0 + + +def test_decorator(tmpdir, httpbin_both): + """Test the decorator version of VCR.py""" + url = httpbin_both.url + + @vcr.use_cassette(str(tmpdir.join("atts.yaml"))) + def inner1(): + resp, _ = http().request(url) + return resp["status"] + + @vcr.use_cassette(str(tmpdir.join("atts.yaml"))) + def inner2(): + resp, _ = http().request(url) + return resp["status"] + + assert inner1() == inner2() diff --git a/tools/vcrpy/tests/integration/test_ignore.py b/tools/vcrpy/tests/integration/test_ignore.py new file mode 100644 index 000000000000..574a67e26e75 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_ignore.py @@ -0,0 +1,67 @@ +from six.moves.urllib.request import urlopen +import socket +from contextlib import contextmanager +import vcr + + +@contextmanager +def overridden_dns(overrides): + """ + Monkeypatch socket.getaddrinfo() to override DNS lookups (name will resolve + to address) + """ + real_getaddrinfo = socket.getaddrinfo + + def fake_getaddrinfo(*args, **kwargs): + if args[0] in overrides: + address = overrides[args[0]] + return [(2, 1, 6, "", (address, args[1]))] + return real_getaddrinfo(*args, **kwargs) + + socket.getaddrinfo = fake_getaddrinfo + yield + socket.getaddrinfo = real_getaddrinfo + + +def test_ignore_localhost(tmpdir, httpbin): + with overridden_dns({"httpbin.org": "127.0.0.1"}): + cass_file = str(tmpdir.join("filter_qs.yaml")) + with vcr.use_cassette(cass_file, ignore_localhost=True) as cass: + urlopen("http://localhost:{}/".format(httpbin.port)) + assert len(cass) == 0 + urlopen("http://httpbin.org:{}/".format(httpbin.port)) + assert len(cass) == 1 + + +def test_ignore_httpbin(tmpdir, httpbin): + with overridden_dns({"httpbin.org": "127.0.0.1"}): + cass_file = str(tmpdir.join("filter_qs.yaml")) + with vcr.use_cassette(cass_file, ignore_hosts=["httpbin.org"]) as cass: + urlopen("http://httpbin.org:{}/".format(httpbin.port)) + assert len(cass) == 0 + urlopen("http://localhost:{}/".format(httpbin.port)) + assert len(cass) == 1 + + +def test_ignore_localhost_and_httpbin(tmpdir, httpbin): + with overridden_dns({"httpbin.org": "127.0.0.1"}): + cass_file = str(tmpdir.join("filter_qs.yaml")) + with vcr.use_cassette(cass_file, ignore_hosts=["httpbin.org"], ignore_localhost=True) as cass: + urlopen("http://httpbin.org:{}".format(httpbin.port)) + urlopen("http://localhost:{}".format(httpbin.port)) + assert len(cass) == 0 + + +def test_ignore_localhost_twice(tmpdir, httpbin): + with overridden_dns({"httpbin.org": "127.0.0.1"}): + cass_file = str(tmpdir.join("filter_qs.yaml")) + with vcr.use_cassette(cass_file, ignore_localhost=True) as cass: + urlopen("http://localhost:{}".format(httpbin.port)) + assert len(cass) == 0 + urlopen("http://httpbin.org:{}".format(httpbin.port)) + assert len(cass) == 1 + with vcr.use_cassette(cass_file, ignore_localhost=True) as cass: + assert len(cass) == 1 + urlopen("http://localhost:{}".format(httpbin.port)) + urlopen("http://httpbin.org:{}".format(httpbin.port)) + assert len(cass) == 1 diff --git a/tools/vcrpy/tests/integration/test_matchers.py b/tools/vcrpy/tests/integration/test_matchers.py new file mode 100644 index 000000000000..9ec28044e7ee --- /dev/null +++ b/tools/vcrpy/tests/integration/test_matchers.py @@ -0,0 +1,107 @@ +import vcr +import pytest +from six.moves.urllib.request import urlopen + + +DEFAULT_URI = "http://httpbin.org/get?p1=q1&p2=q2" # base uri for testing + + +def _replace_httpbin(uri, httpbin, httpbin_secure): + return uri.replace("http://httpbin.org", httpbin.url).replace("https://httpbin.org", httpbin_secure.url) + + +@pytest.fixture +def cassette(tmpdir, httpbin, httpbin_secure): + """ + Helper fixture used to prepare the cassete + returns path to the recorded cassette + """ + default_uri = _replace_httpbin(DEFAULT_URI, httpbin, httpbin_secure) + + cassette_path = str(tmpdir.join("test.yml")) + with vcr.use_cassette(cassette_path, record_mode="all"): + urlopen(default_uri) + return cassette_path + + +@pytest.mark.parametrize( + "matcher, matching_uri, not_matching_uri", + [ + ("uri", "http://httpbin.org/get?p1=q1&p2=q2", "http://httpbin.org/get?p2=q2&p1=q1"), + ("scheme", "http://google.com/post?a=b", "https://httpbin.org/get?p1=q1&p2=q2"), + ("host", "https://httpbin.org/post?a=b", "http://google.com/get?p1=q1&p2=q2"), + ("path", "https://google.com/get?a=b", "http://httpbin.org/post?p1=q1&p2=q2"), + ("query", "https://google.com/get?p2=q2&p1=q1", "http://httpbin.org/get?p1=q1&a=b"), + ], +) +def test_matchers(httpbin, httpbin_secure, cassette, matcher, matching_uri, not_matching_uri): + + matching_uri = _replace_httpbin(matching_uri, httpbin, httpbin_secure) + not_matching_uri = _replace_httpbin(not_matching_uri, httpbin, httpbin_secure) + default_uri = _replace_httpbin(DEFAULT_URI, httpbin, httpbin_secure) + + # play cassette with default uri + with vcr.use_cassette(cassette, match_on=[matcher]) as cass: + urlopen(default_uri) + assert cass.play_count == 1 + + # play cassette with matching on uri + with vcr.use_cassette(cassette, match_on=[matcher]) as cass: + urlopen(matching_uri) + assert cass.play_count == 1 + + # play cassette with not matching on uri, it should fail + with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): + with vcr.use_cassette(cassette, match_on=[matcher]) as cass: + urlopen(not_matching_uri) + + +def test_method_matcher(cassette, httpbin, httpbin_secure): + default_uri = _replace_httpbin(DEFAULT_URI, httpbin, httpbin_secure) + + # play cassette with matching on method + with vcr.use_cassette(cassette, match_on=["method"]) as cass: + urlopen("https://google.com/get?a=b") + assert cass.play_count == 1 + + # should fail if method does not match + with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): + with vcr.use_cassette(cassette, match_on=["method"]) as cass: + # is a POST request + urlopen(default_uri, data=b"") + + +@pytest.mark.parametrize( + "uri", [DEFAULT_URI, "http://httpbin.org/get?p2=q2&p1=q1", "http://httpbin.org/get?p2=q2&p1=q1"] +) +def test_default_matcher_matches(cassette, uri, httpbin, httpbin_secure): + + uri = _replace_httpbin(uri, httpbin, httpbin_secure) + + with vcr.use_cassette(cassette) as cass: + urlopen(uri) + assert cass.play_count == 1 + + +@pytest.mark.parametrize( + "uri", + [ + "https://httpbin.org/get?p1=q1&p2=q2", + "http://google.com/get?p1=q1&p2=q2", + "http://httpbin.org/post?p1=q1&p2=q2", + "http://httpbin.org/get?p1=q1&a=b", + ], +) +def test_default_matcher_does_not_match(cassette, uri, httpbin, httpbin_secure): + uri = _replace_httpbin(uri, httpbin, httpbin_secure) + with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): + with vcr.use_cassette(cassette): + urlopen(uri) + + +def test_default_matcher_does_not_match_on_method(cassette, httpbin, httpbin_secure): + default_uri = _replace_httpbin(DEFAULT_URI, httpbin, httpbin_secure) + with pytest.raises(vcr.errors.CannotOverwriteExistingCassetteException): + with vcr.use_cassette(cassette): + # is a POST request + urlopen(default_uri, data=b"") diff --git a/tools/vcrpy/tests/integration/test_multiple.py b/tools/vcrpy/tests/integration/test_multiple.py new file mode 100644 index 000000000000..0046e08eb130 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_multiple.py @@ -0,0 +1,20 @@ +import pytest +import vcr +from six.moves.urllib.request import urlopen + + +def test_making_extra_request_raises_exception(tmpdir, httpbin): + # make two requests in the first request that are considered + # identical (since the match is based on method) + with vcr.use_cassette(str(tmpdir.join("test.json")), match_on=["method"]): + urlopen(httpbin.url + "/status/200") + urlopen(httpbin.url + "/status/201") + + # Now, try to make three requests. The first two should return the + # correct status codes in order, and the third should raise an + # exception. + with vcr.use_cassette(str(tmpdir.join("test.json")), match_on=["method"]): + assert urlopen(httpbin.url + "/status/200").getcode() == 200 + assert urlopen(httpbin.url + "/status/201").getcode() == 201 + with pytest.raises(Exception): + urlopen(httpbin.url + "/status/200") diff --git a/tools/vcrpy/tests/integration/test_proxy.py b/tools/vcrpy/tests/integration/test_proxy.py new file mode 100644 index 000000000000..31e15a5df5b4 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_proxy.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +"""Test using a proxy.""" + +# External imports +import multiprocessing +import pytest + +from six.moves import socketserver, SimpleHTTPServer +from six.moves.urllib.request import urlopen + +# Internal imports +import vcr + +# Conditional imports +requests = pytest.importorskip("requests") + + +class Proxy(SimpleHTTPServer.SimpleHTTPRequestHandler): + """ + Simple proxy server. + + (Inspired by: http://effbot.org/librarybook/simplehttpserver.htm). + """ + + def do_GET(self): + upstream_response = urlopen(self.path) + try: + status = upstream_response.status + headers = upstream_response.headers.items() + except AttributeError: + # In Python 2 the response is an addinfourl instance. + status = upstream_response.code + headers = upstream_response.info().items() + self.send_response(status, upstream_response.msg) + for header in headers: + self.send_header(*header) + self.end_headers() + self.copyfile(upstream_response, self.wfile) + + +@pytest.yield_fixture(scope="session") +def proxy_server(): + httpd = socketserver.ThreadingTCPServer(("", 0), Proxy) + proxy_process = multiprocessing.Process(target=httpd.serve_forever) + proxy_process.start() + yield "http://{}:{}".format(*httpd.server_address) + proxy_process.terminate() + + +def test_use_proxy(tmpdir, httpbin, proxy_server): + """Ensure that it works with a proxy.""" + with vcr.use_cassette(str(tmpdir.join("proxy.yaml"))): + response = requests.get(httpbin.url, proxies={"http": proxy_server}) + + with vcr.use_cassette(str(tmpdir.join("proxy.yaml"))) as cassette: + cassette_response = requests.get(httpbin.url, proxies={"http": proxy_server}) + + assert cassette_response.headers == response.headers + assert cassette.play_count == 1 diff --git a/tools/vcrpy/tests/integration/test_record_mode.py b/tools/vcrpy/tests/integration/test_record_mode.py new file mode 100644 index 000000000000..7d3ca8d816b1 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_record_mode.py @@ -0,0 +1,142 @@ +import pytest +import vcr +from six.moves.urllib.request import urlopen + + +def test_once_record_mode(tmpdir, httpbin): + testfile = str(tmpdir.join("recordmode.yml")) + with vcr.use_cassette(testfile, record_mode="once"): + # cassette file doesn't exist, so create. + urlopen(httpbin.url).read() + + with vcr.use_cassette(testfile, record_mode="once"): + # make the same request again + urlopen(httpbin.url).read() + + # the first time, it's played from the cassette. + # but, try to access something else from the same cassette, and an + # exception is raised. + with pytest.raises(Exception): + urlopen(httpbin.url + "/get").read() + + +def test_once_record_mode_two_times(tmpdir, httpbin): + testfile = str(tmpdir.join("recordmode.yml")) + with vcr.use_cassette(testfile, record_mode="once"): + # get two of the same file + urlopen(httpbin.url).read() + urlopen(httpbin.url).read() + + with vcr.use_cassette(testfile, record_mode="once"): + # do it again + urlopen(httpbin.url).read() + urlopen(httpbin.url).read() + + +def test_once_mode_three_times(tmpdir, httpbin): + testfile = str(tmpdir.join("recordmode.yml")) + with vcr.use_cassette(testfile, record_mode="once"): + # get three of the same file + urlopen(httpbin.url).read() + urlopen(httpbin.url).read() + urlopen(httpbin.url).read() + + +def test_new_episodes_record_mode(tmpdir, httpbin): + testfile = str(tmpdir.join("recordmode.yml")) + + with vcr.use_cassette(testfile, record_mode="new_episodes"): + # cassette file doesn't exist, so create. + urlopen(httpbin.url).read() + + with vcr.use_cassette(testfile, record_mode="new_episodes") as cass: + # make the same request again + urlopen(httpbin.url).read() + + # all responses have been played + assert cass.all_played + + # in the "new_episodes" record mode, we can add more requests to + # a cassette without repurcussions. + urlopen(httpbin.url + "/get").read() + + # one of the responses has been played + assert cass.play_count == 1 + + # not all responses have been played + assert not cass.all_played + + with vcr.use_cassette(testfile, record_mode="new_episodes") as cass: + # the cassette should now have 2 responses + assert len(cass.responses) == 2 + + +def test_new_episodes_record_mode_two_times(tmpdir, httpbin): + testfile = str(tmpdir.join("recordmode.yml")) + url = httpbin.url + "/bytes/1024" + with vcr.use_cassette(testfile, record_mode="new_episodes"): + # cassette file doesn't exist, so create. + original_first_response = urlopen(url).read() + + with vcr.use_cassette(testfile, record_mode="new_episodes"): + # make the same request again + assert urlopen(url).read() == original_first_response + + # in the "new_episodes" record mode, we can add the same request + # to the cassette without repercussions + original_second_response = urlopen(url).read() + + with vcr.use_cassette(testfile, record_mode="once"): + # make the same request again + assert urlopen(url).read() == original_first_response + assert urlopen(url).read() == original_second_response + # now that we are back in once mode, this should raise + # an error. + with pytest.raises(Exception): + urlopen(url).read() + + +def test_all_record_mode(tmpdir, httpbin): + testfile = str(tmpdir.join("recordmode.yml")) + + with vcr.use_cassette(testfile, record_mode="all"): + # cassette file doesn't exist, so create. + urlopen(httpbin.url).read() + + with vcr.use_cassette(testfile, record_mode="all") as cass: + # make the same request again + urlopen(httpbin.url).read() + + # in the "all" record mode, we can add more requests to + # a cassette without repurcussions. + urlopen(httpbin.url + "/get").read() + + # The cassette was never actually played, even though it existed. + # that's because, in "all" mode, the requests all go directly to + # the source and bypass the cassette. + assert cass.play_count == 0 + + +def test_none_record_mode(tmpdir, httpbin): + # Cassette file doesn't exist, yet we are trying to make a request. + # raise hell. + testfile = str(tmpdir.join("recordmode.yml")) + with vcr.use_cassette(testfile, record_mode="none"): + with pytest.raises(Exception): + urlopen(httpbin.url).read() + + +def test_none_record_mode_with_existing_cassette(tmpdir, httpbin): + # create a cassette file + testfile = str(tmpdir.join("recordmode.yml")) + + with vcr.use_cassette(testfile, record_mode="all"): + urlopen(httpbin.url).read() + + # play from cassette file + with vcr.use_cassette(testfile, record_mode="none") as cass: + urlopen(httpbin.url).read() + assert cass.play_count == 1 + # but if I try to hit the net, raise an exception. + with pytest.raises(Exception): + urlopen(httpbin.url + "/get").read() diff --git a/tools/vcrpy/tests/integration/test_register_matcher.py b/tools/vcrpy/tests/integration/test_register_matcher.py new file mode 100644 index 000000000000..44e44cad1c7d --- /dev/null +++ b/tools/vcrpy/tests/integration/test_register_matcher.py @@ -0,0 +1,36 @@ +import vcr +from six.moves.urllib.request import urlopen + + +def true_matcher(r1, r2): + return True + + +def false_matcher(r1, r2): + return False + + +def test_registered_true_matcher(tmpdir, httpbin): + my_vcr = vcr.VCR() + my_vcr.register_matcher("true", true_matcher) + testfile = str(tmpdir.join("test.yml")) + with my_vcr.use_cassette(testfile, match_on=["true"]): + # These 2 different urls are stored as the same request + urlopen(httpbin.url) + urlopen(httpbin.url + "/get") + + with my_vcr.use_cassette(testfile, match_on=["true"]): + # I can get the response twice even though I only asked for it once + urlopen(httpbin.url + "/get") + urlopen(httpbin.url + "/get") + + +def test_registered_false_matcher(tmpdir, httpbin): + my_vcr = vcr.VCR() + my_vcr.register_matcher("false", false_matcher) + testfile = str(tmpdir.join("test.yml")) + with my_vcr.use_cassette(testfile, match_on=["false"]) as cass: + # These 2 different urls are stored as different requests + urlopen(httpbin.url) + urlopen(httpbin.url + "/get") + assert len(cass) == 2 diff --git a/tools/vcrpy/tests/integration/test_register_persister.py b/tools/vcrpy/tests/integration/test_register_persister.py new file mode 100644 index 000000000000..493b45297ae2 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_register_persister.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +"""Tests for cassettes with custom persistence""" + +# External imports +import os +from six.moves.urllib.request import urlopen + +# Internal imports +import vcr +from vcr.persisters.filesystem import FilesystemPersister + + +class CustomFilesystemPersister(object): + """Behaves just like default FilesystemPersister but adds .test extension + to the cassette file""" + + @staticmethod + def load_cassette(cassette_path, serializer): + cassette_path += ".test" + return FilesystemPersister.load_cassette(cassette_path, serializer) + + @staticmethod + def save_cassette(cassette_path, cassette_dict, serializer): + cassette_path += ".test" + FilesystemPersister.save_cassette(cassette_path, cassette_dict, serializer) + + +def test_save_cassette_with_custom_persister(tmpdir, httpbin): + """Ensure you can save a cassette using custom persister""" + my_vcr = vcr.VCR() + my_vcr.register_persister(CustomFilesystemPersister) + + # Check to make sure directory doesnt exist + assert not os.path.exists(str(tmpdir.join("nonexistent"))) + + # Run VCR to create dir and cassette file using new save_cassette callback + with my_vcr.use_cassette(str(tmpdir.join("nonexistent", "cassette.yml"))): + urlopen(httpbin.url).read() + + # Callback should have made the file and the directory + assert os.path.exists(str(tmpdir.join("nonexistent", "cassette.yml.test"))) + + +def test_load_cassette_with_custom_persister(tmpdir, httpbin): + """ + Ensure you can load a cassette using custom persister + """ + my_vcr = vcr.VCR() + my_vcr.register_persister(CustomFilesystemPersister) + + test_fixture = str(tmpdir.join("synopsis.json.test")) + + with my_vcr.use_cassette(test_fixture, serializer="json"): + response = urlopen(httpbin.url).read() + assert b"difficult sometimes" in response diff --git a/tools/vcrpy/tests/integration/test_register_serializer.py b/tools/vcrpy/tests/integration/test_register_serializer.py new file mode 100644 index 000000000000..9e698de216e6 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_register_serializer.py @@ -0,0 +1,33 @@ +import vcr + + +class MockSerializer(object): + def __init__(self): + self.serialize_count = 0 + self.deserialize_count = 0 + self.load_args = None + + def deserialize(self, cassette_string): + self.serialize_count += 1 + self.cassette_string = cassette_string + return {"interactions": []} + + def serialize(self, cassette_dict): + self.deserialize_count += 1 + return "" + + +def test_registered_serializer(tmpdir): + ms = MockSerializer() + my_vcr = vcr.VCR() + my_vcr.register_serializer("mock", ms) + tmpdir.join("test.mock").write("test_data") + with my_vcr.use_cassette(str(tmpdir.join("test.mock")), serializer="mock"): + # Serializer deserialized once + assert ms.serialize_count == 1 + # and serialized the test data string + assert ms.cassette_string == "test_data" + # and hasn't serialized yet + assert ms.deserialize_count == 0 + + assert ms.serialize_count == 1 diff --git a/tools/vcrpy/tests/integration/test_request.py b/tools/vcrpy/tests/integration/test_request.py new file mode 100644 index 000000000000..04f7385a7046 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_request.py @@ -0,0 +1,19 @@ +import vcr +from six.moves.urllib.request import urlopen + + +def test_recorded_request_uri_with_redirected_request(tmpdir, httpbin): + with vcr.use_cassette(str(tmpdir.join("test.yml"))) as cass: + assert len(cass) == 0 + urlopen(httpbin.url + "/redirect/3") + assert cass.requests[0].uri == httpbin.url + "/redirect/3" + assert cass.requests[3].uri == httpbin.url + "/get" + assert len(cass) == 4 + + +def test_records_multiple_header_values(tmpdir, httpbin): + with vcr.use_cassette(str(tmpdir.join("test.yml"))) as cass: + assert len(cass) == 0 + urlopen(httpbin.url + "/response-headers?foo=bar&foo=baz") + assert len(cass) == 1 + assert cass.responses[0]["headers"]["foo"] == ["bar", "baz"] diff --git a/tools/vcrpy/tests/integration/test_requests.py b/tools/vcrpy/tests/integration/test_requests.py new file mode 100644 index 000000000000..8665bdf2e759 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_requests.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- +"""Test requests' interaction with vcr""" +import platform +import pytest +import sys +import vcr +from assertions import assert_cassette_empty, assert_is_json + +requests = pytest.importorskip("requests") +from requests.exceptions import ConnectionError # noqa E402 + + +def test_status_code(httpbin_both, tmpdir): + """Ensure that we can read the status code""" + url = httpbin_both.url + "/" + with vcr.use_cassette(str(tmpdir.join("atts.yaml"))): + status_code = requests.get(url).status_code + + with vcr.use_cassette(str(tmpdir.join("atts.yaml"))): + assert status_code == requests.get(url).status_code + + +def test_headers(httpbin_both, tmpdir): + """Ensure that we can read the headers back""" + url = httpbin_both + "/" + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + headers = requests.get(url).headers + + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + assert headers == requests.get(url).headers + + +def test_body(tmpdir, httpbin_both): + """Ensure the responses are all identical enough""" + url = httpbin_both + "/bytes/1024" + with vcr.use_cassette(str(tmpdir.join("body.yaml"))): + content = requests.get(url).content + + with vcr.use_cassette(str(tmpdir.join("body.yaml"))): + assert content == requests.get(url).content + + +def test_get_empty_content_type_json(tmpdir, httpbin_both): + """Ensure GET with application/json content-type and empty request body doesn't crash""" + url = httpbin_both + "/status/200" + headers = {"Content-Type": "application/json"} + + with vcr.use_cassette(str(tmpdir.join("get_empty_json.yaml")), match_on=("body",)): + status = requests.get(url, headers=headers).status_code + + with vcr.use_cassette(str(tmpdir.join("get_empty_json.yaml")), match_on=("body",)): + assert status == requests.get(url, headers=headers).status_code + + +def test_effective_url(tmpdir, httpbin_both): + """Ensure that the effective_url is captured""" + url = httpbin_both.url + "/redirect-to?url=/html" + with vcr.use_cassette(str(tmpdir.join("url.yaml"))): + effective_url = requests.get(url).url + assert effective_url == httpbin_both.url + "/html" + + with vcr.use_cassette(str(tmpdir.join("url.yaml"))): + assert effective_url == requests.get(url).url + + +def test_auth(tmpdir, httpbin_both): + """Ensure that we can handle basic auth""" + auth = ("user", "passwd") + url = httpbin_both + "/basic-auth/user/passwd" + with vcr.use_cassette(str(tmpdir.join("auth.yaml"))): + one = requests.get(url, auth=auth) + + with vcr.use_cassette(str(tmpdir.join("auth.yaml"))): + two = requests.get(url, auth=auth) + assert one.content == two.content + assert one.status_code == two.status_code + + +def test_auth_failed(tmpdir, httpbin_both): + """Ensure that we can save failed auth statuses""" + auth = ("user", "wrongwrongwrong") + url = httpbin_both + "/basic-auth/user/passwd" + with vcr.use_cassette(str(tmpdir.join("auth-failed.yaml"))) as cass: + # Ensure that this is empty to begin with + assert_cassette_empty(cass) + one = requests.get(url, auth=auth) + two = requests.get(url, auth=auth) + assert one.content == two.content + assert one.status_code == two.status_code == 401 + + +def test_post(tmpdir, httpbin_both): + """Ensure that we can post and cache the results""" + data = {"key1": "value1", "key2": "value2"} + url = httpbin_both + "/post" + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))): + req1 = requests.post(url, data).content + + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))): + req2 = requests.post(url, data).content + + assert req1 == req2 + + +def test_post_chunked_binary(tmpdir, httpbin): + """Ensure that we can send chunked binary without breaking while trying to concatenate bytes with str.""" + data1 = iter([b"data", b"to", b"send"]) + data2 = iter([b"data", b"to", b"send"]) + url = httpbin.url + "/post" + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))): + req1 = requests.post(url, data1).content + + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))): + req2 = requests.post(url, data2).content + + assert req1 == req2 + + +@pytest.mark.skipif("sys.version_info >= (3, 6)", strict=True, raises=ConnectionError) +@pytest.mark.skipif( + (3, 5) < sys.version_info < (3, 6) and platform.python_implementation() == "CPython", + reason="Fails on CPython 3.5", +) +def test_post_chunked_binary_secure(tmpdir, httpbin_secure): + """Ensure that we can send chunked binary without breaking while trying to concatenate bytes with str.""" + data1 = iter([b"data", b"to", b"send"]) + data2 = iter([b"data", b"to", b"send"]) + url = httpbin_secure.url + "/post" + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))): + req1 = requests.post(url, data1).content + print(req1) + + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))): + req2 = requests.post(url, data2).content + + assert req1 == req2 + + +def test_redirects(tmpdir, httpbin_both): + """Ensure that we can handle redirects""" + url = httpbin_both + "/redirect-to?url=bytes/1024" + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))): + content = requests.get(url).content + + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))) as cass: + assert content == requests.get(url).content + # Ensure that we've now cached *two* responses. One for the redirect + # and one for the final fetch + assert len(cass) == 2 + assert cass.play_count == 2 + + +def test_cross_scheme(tmpdir, httpbin_secure, httpbin): + """Ensure that requests between schemes are treated separately""" + # First fetch a url under http, and then again under https and then + # ensure that we haven't served anything out of cache, and we have two + # requests / response pairs in the cassette + with vcr.use_cassette(str(tmpdir.join("cross_scheme.yaml"))) as cass: + requests.get(httpbin_secure + "/") + requests.get(httpbin + "/") + assert cass.play_count == 0 + assert len(cass) == 2 + + +def test_gzip(tmpdir, httpbin_both): + """ + Ensure that requests (actually urllib3) is able to automatically decompress + the response body + """ + url = httpbin_both + "/gzip" + response = requests.get(url) + + with vcr.use_cassette(str(tmpdir.join("gzip.yaml"))): + response = requests.get(url) + assert_is_json(response.content) + + with vcr.use_cassette(str(tmpdir.join("gzip.yaml"))): + assert_is_json(response.content) + + +def test_session_and_connection_close(tmpdir, httpbin): + """ + This tests the issue in https://github.com/kevin1024/vcrpy/issues/48 + + If you use a requests.session and the connection is closed, then an + exception is raised in the urllib3 module vendored into requests: + `AttributeError: 'NoneType' object has no attribute 'settimeout'` + """ + with vcr.use_cassette(str(tmpdir.join("session_connection_closed.yaml"))): + session = requests.session() + + session.get(httpbin + "/get", headers={"Connection": "close"}) + session.get(httpbin + "/get", headers={"Connection": "close"}) + + +def test_https_with_cert_validation_disabled(tmpdir, httpbin_secure): + with vcr.use_cassette(str(tmpdir.join("cert_validation_disabled.yaml"))): + requests.get(httpbin_secure.url, verify=False) + + +def test_session_can_make_requests_after_requests_unpatched(tmpdir, httpbin): + with vcr.use_cassette(str(tmpdir.join("test_session_after_unpatched.yaml"))): + session = requests.session() + session.get(httpbin + "/get") + + with vcr.use_cassette(str(tmpdir.join("test_session_after_unpatched.yaml"))): + session = requests.session() + session.get(httpbin + "/get") + + session.get(httpbin + "/status/200") + + +def test_session_created_before_use_cassette_is_patched(tmpdir, httpbin_both): + url = httpbin_both + "/bytes/1024" + # Record arbitrary, random data to the cassette + with vcr.use_cassette(str(tmpdir.join("session_created_outside.yaml"))): + session = requests.session() + body = session.get(url).content + + # Create a session outside of any cassette context manager + session = requests.session() + # Make a request to make sure that a connectionpool is instantiated + session.get(httpbin_both + "/get") + + with vcr.use_cassette(str(tmpdir.join("session_created_outside.yaml"))): + # These should only be the same if the patching succeeded. + assert session.get(url).content == body + + +def test_nested_cassettes_with_session_created_before_nesting(httpbin_both, tmpdir): + """ + This tests ensures that a session that was created while one cassette was + active is patched to the use the responses of a second cassette when it + is enabled. + """ + url = httpbin_both + "/bytes/1024" + with vcr.use_cassette(str(tmpdir.join("first_nested.yaml"))): + session = requests.session() + first_body = session.get(url).content + with vcr.use_cassette(str(tmpdir.join("second_nested.yaml"))): + second_body = session.get(url).content + third_body = requests.get(url).content + + with vcr.use_cassette(str(tmpdir.join("second_nested.yaml"))): + session = requests.session() + assert session.get(url).content == second_body + with vcr.use_cassette(str(tmpdir.join("first_nested.yaml"))): + assert session.get(url).content == first_body + assert session.get(url).content == third_body + + # Make sure that the session can now get content normally. + assert "User-agent" in session.get(httpbin_both.url + "/robots.txt").text + + +def test_post_file(tmpdir, httpbin_both): + """Ensure that we handle posting a file.""" + url = httpbin_both + "/post" + with vcr.use_cassette(str(tmpdir.join("post_file.yaml"))) as cass, open("tox.ini", "rb") as f: + original_response = requests.post(url, f).content + + # This also tests that we do the right thing with matching the body when they are files. + with vcr.use_cassette( + str(tmpdir.join("post_file.yaml")), + match_on=("method", "scheme", "host", "port", "path", "query", "body"), + ) as cass: + with open("tox.ini", "rb") as f: + tox_content = f.read() + assert cass.requests[0].body.read() == tox_content + with open("tox.ini", "rb") as f: + new_response = requests.post(url, f).content + assert original_response == new_response + + +def test_filter_post_params(tmpdir, httpbin_both): + """ + This tests the issue in https://github.com/kevin1024/vcrpy/issues/158 + + Ensure that a post request made through requests can still be filtered. + with vcr.use_cassette(cass_file, filter_post_data_parameters=['id']) as cass: + assert b'id=secret' not in cass.requests[0].body + """ + url = httpbin_both.url + "/post" + cass_loc = str(tmpdir.join("filter_post_params.yaml")) + with vcr.use_cassette(cass_loc, filter_post_data_parameters=["key"]) as cass: + requests.post(url, data={"key": "value"}) + with vcr.use_cassette(cass_loc, filter_post_data_parameters=["key"]) as cass: + assert b"key=value" not in cass.requests[0].body + + +def test_post_unicode_match_on_body(tmpdir, httpbin_both): + """Ensure that matching on POST body that contains Unicode characters works.""" + data = {"key1": "value1", "●‿●": "٩(●̮̮̃•̃)۶"} + url = httpbin_both + "/post" + + with vcr.use_cassette(str(tmpdir.join("requests.yaml")), additional_matchers=("body",)): + req1 = requests.post(url, data).content + + with vcr.use_cassette(str(tmpdir.join("requests.yaml")), additional_matchers=("body",)): + req2 = requests.post(url, data).content + + assert req1 == req2 diff --git a/tools/vcrpy/tests/integration/test_stubs.py b/tools/vcrpy/tests/integration/test_stubs.py new file mode 100644 index 000000000000..4c24d064e07e --- /dev/null +++ b/tools/vcrpy/tests/integration/test_stubs.py @@ -0,0 +1,134 @@ +import vcr +import zlib +import json +import six.moves.http_client as httplib + +from assertions import assert_is_json + + +def _headers_are_case_insensitive(host, port): + conn = httplib.HTTPConnection(host, port) + conn.request("GET", "/cookies/set?k1=v1") + r1 = conn.getresponse() + cookie_data1 = r1.getheader("set-cookie") + conn = httplib.HTTPConnection(host, port) + conn.request("GET", "/cookies/set?k1=v1") + r2 = conn.getresponse() + cookie_data2 = r2.getheader("Set-Cookie") + return cookie_data1 == cookie_data2 + + +def test_case_insensitivity(tmpdir, httpbin): + testfile = str(tmpdir.join("case_insensitivity.yml")) + # check if headers are case insensitive outside of vcrpy + host, port = httpbin.host, httpbin.port + outside = _headers_are_case_insensitive(host, port) + with vcr.use_cassette(testfile): + # check if headers are case insensitive inside of vcrpy + inside = _headers_are_case_insensitive(host, port) + # check if headers are case insensitive after vcrpy deserializes headers + inside2 = _headers_are_case_insensitive(host, port) + + # behavior should be the same both inside and outside + assert outside == inside == inside2 + + +def _multiple_header_value(httpbin): + conn = httplib.HTTPConnection(httpbin.host, httpbin.port) + conn.request("GET", "/response-headers?foo=bar&foo=baz") + r = conn.getresponse() + return r.getheader("foo") + + +def test_multiple_headers(tmpdir, httpbin): + testfile = str(tmpdir.join("multiple_headers.yaml")) + outside = _multiple_header_value(httpbin) + + with vcr.use_cassette(testfile): + inside = _multiple_header_value(httpbin) + + assert outside == inside + + +def test_original_decoded_response_is_not_modified(tmpdir, httpbin): + testfile = str(tmpdir.join("decoded_response.yml")) + host, port = httpbin.host, httpbin.port + + conn = httplib.HTTPConnection(host, port) + conn.request("GET", "/gzip") + outside = conn.getresponse() + + with vcr.use_cassette(testfile, decode_compressed_response=True): + conn = httplib.HTTPConnection(host, port) + conn.request("GET", "/gzip") + inside = conn.getresponse() + + # Assert that we do not modify the original response while appending + # to the casssette. + assert "gzip" == inside.headers["content-encoding"] + + # They should effectively be the same response. + inside_headers = (h for h in inside.headers.items() if h[0].lower() != "date") + outside_headers = (h for h in outside.getheaders() if h[0].lower() != "date") + assert set(inside_headers) == set(outside_headers) + inside = zlib.decompress(inside.read(), 16 + zlib.MAX_WBITS) + outside = zlib.decompress(outside.read(), 16 + zlib.MAX_WBITS) + assert inside == outside + + # Even though the above are raw bytes, the JSON data should have been + # decoded and saved to the cassette. + with vcr.use_cassette(testfile): + conn = httplib.HTTPConnection(host, port) + conn.request("GET", "/gzip") + inside = conn.getresponse() + + assert "content-encoding" not in inside.headers + assert_is_json(inside.read()) + + +def _make_before_record_response(fields, replacement="[REDACTED]"): + def before_record_response(response): + string_body = response["body"]["string"].decode("utf8") + body = json.loads(string_body) + + for field in fields: + if field in body: + body[field] = replacement + + response["body"]["string"] = json.dumps(body).encode() + return response + + return before_record_response + + +def test_original_response_is_not_modified_by_before_filter(tmpdir, httpbin): + testfile = str(tmpdir.join("sensitive_data_scrubbed_response.yml")) + host, port = httpbin.host, httpbin.port + field_to_scrub = "url" + replacement = "[YOU_CANT_HAVE_THE_MANGO]" + + conn = httplib.HTTPConnection(host, port) + conn.request("GET", "/get") + outside = conn.getresponse() + + callback = _make_before_record_response([field_to_scrub], replacement) + with vcr.use_cassette(testfile, before_record_response=callback): + conn = httplib.HTTPConnection(host, port) + conn.request("GET", "/get") + inside = conn.getresponse() + + # The scrubbed field should be the same, because no cassette existed. + # Furthermore, the responses should be identical. + inside_body = json.loads(inside.read().decode("utf-8")) + outside_body = json.loads(outside.read().decode("utf-8")) + assert not inside_body[field_to_scrub] == replacement + assert inside_body[field_to_scrub] == outside_body[field_to_scrub] + + # Ensure that when a cassette exists, the scrubbed response is returned. + with vcr.use_cassette(testfile, before_record_response=callback): + conn = httplib.HTTPConnection(host, port) + conn.request("GET", "/get") + inside = conn.getresponse() + + inside_body = json.loads(inside.read().decode("utf-8")) + assert inside_body[field_to_scrub] == replacement diff --git a/tools/vcrpy/tests/integration/test_tornado.py b/tools/vcrpy/tests/integration/test_tornado.py new file mode 100644 index 000000000000..327037966aae --- /dev/null +++ b/tools/vcrpy/tests/integration/test_tornado.py @@ -0,0 +1,350 @@ +# -*- coding: utf-8 -*- +"""Test requests' interaction with vcr""" + +import json + +import pytest +import vcr +from vcr.errors import CannotOverwriteExistingCassetteException + +from assertions import assert_cassette_empty, assert_is_json + +tornado = pytest.importorskip("tornado") +http = pytest.importorskip("tornado.httpclient") + +# whether the current version of Tornado supports the raise_error argument for +# fetch(). +supports_raise_error = tornado.version_info >= (4,) + + +@pytest.fixture(params=["simple", "curl", "default"]) +def get_client(request): + if request.param == "simple": + from tornado import simple_httpclient as simple + + return lambda: simple.SimpleAsyncHTTPClient() + elif request.param == "curl": + curl = pytest.importorskip("tornado.curl_httpclient") + return lambda: curl.CurlAsyncHTTPClient() + else: + return lambda: http.AsyncHTTPClient() + + +def get(client, url, **kwargs): + fetch_kwargs = {} + if supports_raise_error: + fetch_kwargs["raise_error"] = kwargs.pop("raise_error", True) + + return client.fetch(http.HTTPRequest(url, method="GET", **kwargs), **fetch_kwargs) + + +def post(client, url, data=None, **kwargs): + if data: + kwargs["body"] = json.dumps(data) + return client.fetch(http.HTTPRequest(url, method="POST", **kwargs)) + + +@pytest.fixture(params=["https", "http"]) +def scheme(request): + """Fixture that returns both http and https.""" + return request.param + + +@pytest.mark.gen_test +def test_status_code(get_client, scheme, tmpdir): + """Ensure that we can read the status code""" + url = scheme + "://httpbin.org/" + with vcr.use_cassette(str(tmpdir.join("atts.yaml"))): + status_code = (yield get(get_client(), url)).code + + with vcr.use_cassette(str(tmpdir.join("atts.yaml"))) as cass: + assert status_code == (yield get(get_client(), url)).code + assert 1 == cass.play_count + + +@pytest.mark.gen_test +def test_headers(get_client, scheme, tmpdir): + """Ensure that we can read the headers back""" + url = scheme + "://httpbin.org/" + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + headers = (yield get(get_client(), url)).headers + + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))) as cass: + assert headers == (yield get(get_client(), url)).headers + assert 1 == cass.play_count + + +@pytest.mark.gen_test +def test_body(get_client, tmpdir, scheme): + """Ensure the responses are all identical enough""" + + url = scheme + "://httpbin.org/bytes/1024" + with vcr.use_cassette(str(tmpdir.join("body.yaml"))): + content = (yield get(get_client(), url)).body + + with vcr.use_cassette(str(tmpdir.join("body.yaml"))) as cass: + assert content == (yield get(get_client(), url)).body + assert 1 == cass.play_count + + +@pytest.mark.gen_test +def test_effective_url(get_client, scheme, tmpdir): + """Ensure that the effective_url is captured""" + url = scheme + "://httpbin.org/redirect-to?url=/html" + with vcr.use_cassette(str(tmpdir.join("url.yaml"))): + effective_url = (yield get(get_client(), url)).effective_url + assert effective_url == scheme + "://httpbin.org/html" + + with vcr.use_cassette(str(tmpdir.join("url.yaml"))) as cass: + assert effective_url == (yield get(get_client(), url)).effective_url + assert 1 == cass.play_count + + +@pytest.mark.gen_test +def test_auth(get_client, tmpdir, scheme): + """Ensure that we can handle basic auth""" + auth = ("user", "passwd") + url = scheme + "://httpbin.org/basic-auth/user/passwd" + with vcr.use_cassette(str(tmpdir.join("auth.yaml"))): + one = yield get(get_client(), url, auth_username=auth[0], auth_password=auth[1]) + + with vcr.use_cassette(str(tmpdir.join("auth.yaml"))) as cass: + two = yield get(get_client(), url, auth_username=auth[0], auth_password=auth[1]) + assert one.body == two.body + assert one.code == two.code + assert 1 == cass.play_count + + +@pytest.mark.gen_test +def test_auth_failed(get_client, tmpdir, scheme): + """Ensure that we can save failed auth statuses""" + auth = ("user", "wrongwrongwrong") + url = scheme + "://httpbin.org/basic-auth/user/passwd" + with vcr.use_cassette(str(tmpdir.join("auth-failed.yaml"))) as cass: + # Ensure that this is empty to begin with + assert_cassette_empty(cass) + with pytest.raises(http.HTTPError) as exc_info: + yield get(get_client(), url, auth_username=auth[0], auth_password=auth[1]) + one = exc_info.value.response + assert exc_info.value.code == 401 + + with vcr.use_cassette(str(tmpdir.join("auth-failed.yaml"))) as cass: + with pytest.raises(http.HTTPError) as exc_info: + two = yield get(get_client(), url, auth_username=auth[0], auth_password=auth[1]) + two = exc_info.value.response + assert exc_info.value.code == 401 + assert one.body == two.body + assert one.code == two.code == 401 + assert 1 == cass.play_count + + +@pytest.mark.gen_test +def test_post(get_client, tmpdir, scheme): + """Ensure that we can post and cache the results""" + data = {"key1": "value1", "key2": "value2"} + url = scheme + "://httpbin.org/post" + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))): + req1 = (yield post(get_client(), url, data)).body + + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))) as cass: + req2 = (yield post(get_client(), url, data)).body + + assert req1 == req2 + assert 1 == cass.play_count + + +@pytest.mark.gen_test +def test_redirects(get_client, tmpdir, scheme): + """Ensure that we can handle redirects""" + url = scheme + "://httpbin.org/redirect-to?url=bytes/1024" + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))): + content = (yield get(get_client(), url)).body + + with vcr.use_cassette(str(tmpdir.join("requests.yaml"))) as cass: + assert content == (yield get(get_client(), url)).body + assert cass.play_count == 1 + + +@pytest.mark.gen_test +def test_cross_scheme(get_client, tmpdir, scheme): + """Ensure that requests between schemes are treated separately""" + # First fetch a url under http, and then again under https and then + # ensure that we haven't served anything out of cache, and we have two + # requests / response pairs in the cassette + with vcr.use_cassette(str(tmpdir.join("cross_scheme.yaml"))) as cass: + yield get(get_client(), "https://httpbin.org/") + yield get(get_client(), "http://httpbin.org/") + assert cass.play_count == 0 + assert len(cass) == 2 + + # Then repeat the same requests and ensure both were replayed. + with vcr.use_cassette(str(tmpdir.join("cross_scheme.yaml"))) as cass: + yield get(get_client(), "https://httpbin.org/") + yield get(get_client(), "http://httpbin.org/") + assert cass.play_count == 2 + + +@pytest.mark.gen_test +def test_gzip(get_client, tmpdir, scheme): + """ + Ensure that httpclient is able to automatically decompress the response + body + """ + url = scheme + "://httpbin.org/gzip" + + # use_gzip was renamed to decompress_response in 4.0 + kwargs = {} + if tornado.version_info < (4,): + kwargs["use_gzip"] = True + else: + kwargs["decompress_response"] = True + + with vcr.use_cassette(str(tmpdir.join("gzip.yaml"))): + response = yield get(get_client(), url, **kwargs) + assert_is_json(response.body) + + with vcr.use_cassette(str(tmpdir.join("gzip.yaml"))) as cass: + response = yield get(get_client(), url, **kwargs) + assert_is_json(response.body) + assert 1 == cass.play_count + + +@pytest.mark.gen_test +def test_https_with_cert_validation_disabled(get_client, tmpdir): + cass_path = str(tmpdir.join("cert_validation_disabled.yaml")) + + with vcr.use_cassette(cass_path): + yield get(get_client(), "https://httpbin.org", validate_cert=False) + + with vcr.use_cassette(cass_path) as cass: + yield get(get_client(), "https://httpbin.org", validate_cert=False) + assert 1 == cass.play_count + + +@pytest.mark.gen_test +def test_unsupported_features_raises_in_future(get_client, tmpdir): + """Ensure that the exception for an AsyncHTTPClient feature not being + supported is raised inside the future.""" + + def callback(chunk): + assert False, "Did not expect to be called." + + with vcr.use_cassette(str(tmpdir.join("invalid.yaml"))): + future = get(get_client(), "http://httpbin.org", streaming_callback=callback) + + with pytest.raises(Exception) as excinfo: + yield future + + assert "not yet supported by VCR" in str(excinfo) + + +@pytest.mark.skipif(not supports_raise_error, reason="raise_error unavailable in tornado <= 3") +@pytest.mark.gen_test +def test_unsupported_features_raise_error_disabled(get_client, tmpdir): + """Ensure that the exception for an AsyncHTTPClient feature not being + supported is not raised if raise_error=False.""" + + def callback(chunk): + assert False, "Did not expect to be called." + + with vcr.use_cassette(str(tmpdir.join("invalid.yaml"))): + response = yield get( + get_client(), "http://httpbin.org", streaming_callback=callback, raise_error=False + ) + + assert "not yet supported by VCR" in str(response.error) + + +@pytest.mark.gen_test +def test_cannot_overwrite_cassette_raises_in_future(get_client, tmpdir): + """Ensure that CannotOverwriteExistingCassetteException is raised inside + the future.""" + + with vcr.use_cassette(str(tmpdir.join("overwrite.yaml"))): + yield get(get_client(), "http://httpbin.org/get") + + with vcr.use_cassette(str(tmpdir.join("overwrite.yaml"))): + future = get(get_client(), "http://httpbin.org/headers") + + with pytest.raises(CannotOverwriteExistingCassetteException): + yield future + + +@pytest.mark.skipif(not supports_raise_error, reason="raise_error unavailable in tornado <= 3") +@pytest.mark.gen_test +def test_cannot_overwrite_cassette_raise_error_disabled(get_client, tmpdir): + """Ensure that CannotOverwriteExistingCassetteException is not raised if + raise_error=False in the fetch() call.""" + + with vcr.use_cassette(str(tmpdir.join("overwrite.yaml"))): + yield get(get_client(), "http://httpbin.org/get", raise_error=False) + + with vcr.use_cassette(str(tmpdir.join("overwrite.yaml"))): + response = yield get(get_client(), "http://httpbin.org/headers", raise_error=False) + + assert isinstance(response.error, CannotOverwriteExistingCassetteException) + + +@pytest.mark.gen_test +@vcr.use_cassette(path_transformer=vcr.default_vcr.ensure_suffix(".yaml")) +def test_tornado_with_decorator_use_cassette(get_client): + response = yield get_client().fetch(http.HTTPRequest("http://www.google.com/", method="GET")) + assert response.body.decode("utf-8") == "not actually google" + + +@pytest.mark.gen_test +@vcr.use_cassette(path_transformer=vcr.default_vcr.ensure_suffix(".yaml")) +def test_tornado_exception_can_be_caught(get_client): + try: + yield get(get_client(), "http://httpbin.org/status/500") + except http.HTTPError as e: + assert e.code == 500 + + try: + yield get(get_client(), "http://httpbin.org/status/404") + except http.HTTPError as e: + assert e.code == 404 + + +@pytest.mark.gen_test +def test_existing_references_get_patched(tmpdir): + from tornado.httpclient import AsyncHTTPClient + + with vcr.use_cassette(str(tmpdir.join("data.yaml"))): + client = AsyncHTTPClient() + yield get(client, "http://httpbin.org/get") + + with vcr.use_cassette(str(tmpdir.join("data.yaml"))) as cass: + yield get(client, "http://httpbin.org/get") + assert cass.play_count == 1 + + +@pytest.mark.gen_test +def test_existing_instances_get_patched(get_client, tmpdir): + """Ensure that existing instances of AsyncHTTPClient get patched upon + entering VCR context.""" + + client = get_client() + + with vcr.use_cassette(str(tmpdir.join("data.yaml"))): + yield get(client, "http://httpbin.org/get") + + with vcr.use_cassette(str(tmpdir.join("data.yaml"))) as cass: + yield get(client, "http://httpbin.org/get") + assert cass.play_count == 1 + + +@pytest.mark.gen_test +def test_request_time_is_set(get_client, tmpdir): + """Ensures that the request_time on HTTPResponses is set.""" + + with vcr.use_cassette(str(tmpdir.join("data.yaml"))): + client = get_client() + response = yield get(client, "http://httpbin.org/get") + assert response.request_time is not None + + with vcr.use_cassette(str(tmpdir.join("data.yaml"))) as cass: + client = get_client() + response = yield get(client, "http://httpbin.org/get") + assert response.request_time is not None + assert cass.play_count == 1 diff --git a/tools/vcrpy/tests/integration/test_tornado_exception_can_be_caught.yaml b/tools/vcrpy/tests/integration/test_tornado_exception_can_be_caught.yaml new file mode 100644 index 000000000000..c88f1f02f2e0 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_tornado_exception_can_be_caught.yaml @@ -0,0 +1,62 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://httpbin.org/status/500 + response: + body: {string: !!python/unicode ''} + headers: + - !!python/tuple + - Content-Length + - ['0'] + - !!python/tuple + - Server + - [nginx] + - !!python/tuple + - Connection + - [close] + - !!python/tuple + - Access-Control-Allow-Credentials + - ['true'] + - !!python/tuple + - Date + - ['Thu, 30 Jul 2015 17:32:39 GMT'] + - !!python/tuple + - Access-Control-Allow-Origin + - ['*'] + - !!python/tuple + - Content-Type + - [text/html; charset=utf-8] + status: {code: 500, message: INTERNAL SERVER ERROR} +- request: + body: null + headers: {} + method: GET + uri: http://httpbin.org/status/404 + response: + body: {string: !!python/unicode ''} + headers: + - !!python/tuple + - Content-Length + - ['0'] + - !!python/tuple + - Server + - [nginx] + - !!python/tuple + - Connection + - [close] + - !!python/tuple + - Access-Control-Allow-Credentials + - ['true'] + - !!python/tuple + - Date + - ['Thu, 30 Jul 2015 17:32:39 GMT'] + - !!python/tuple + - Access-Control-Allow-Origin + - ['*'] + - !!python/tuple + - Content-Type + - [text/html; charset=utf-8] + status: {code: 404, message: NOT FOUND} +version: 1 diff --git a/tools/vcrpy/tests/integration/test_tornado_with_decorator_use_cassette.yaml b/tools/vcrpy/tests/integration/test_tornado_with_decorator_use_cassette.yaml new file mode 100644 index 000000000000..ae05aca28351 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_tornado_with_decorator_use_cassette.yaml @@ -0,0 +1,53 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: http://www.google.com/ + response: + body: {string: !!python/unicode 'not actually google'} + headers: + - !!python/tuple + - Expires + - ['-1'] + - !!python/tuple + - Connection + - [close] + - !!python/tuple + - P3p + - ['CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 + for more info."'] + - !!python/tuple + - Alternate-Protocol + - ['80:quic,p=0'] + - !!python/tuple + - Accept-Ranges + - [none] + - !!python/tuple + - X-Xss-Protection + - [1; mode=block] + - !!python/tuple + - Vary + - [Accept-Encoding] + - !!python/tuple + - Date + - ['Thu, 30 Jul 2015 08:41:40 GMT'] + - !!python/tuple + - Cache-Control + - ['private, max-age=0'] + - !!python/tuple + - Content-Type + - [text/html; charset=ISO-8859-1] + - !!python/tuple + - Set-Cookie + - ['PREF=ID=1111111111111111:FF=0:TM=1438245700:LM=1438245700:V=1:S=GAzVO0ALebSpC_cJ; + expires=Sat, 29-Jul-2017 08:41:40 GMT; path=/; domain=.google.com', 'NID=69=Br7oRAwgmKoK__HC6FEnuxglTFDmFxqP6Md63lKhzW1w6WkDbp3U90CDxnUKvDP6wJH8yxY5Lk5ZnFf66Q1B0d4OsYoKgq0vjfBAYXuCIAWtOuGZEOsFXanXs7pt2Mjx; + expires=Fri, 29-Jan-2016 08:41:40 GMT; path=/; domain=.google.com; HttpOnly'] + - !!python/tuple + - X-Frame-Options + - [SAMEORIGIN] + - !!python/tuple + - Server + - [gws] + status: {code: 200, message: OK} +version: 1 diff --git a/tools/vcrpy/tests/integration/test_urllib2.py b/tools/vcrpy/tests/integration/test_urllib2.py new file mode 100644 index 000000000000..ec030ec6e843 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_urllib2.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +"""Integration tests with urllib2""" + +import ssl +from six.moves.urllib.request import urlopen +from six.moves.urllib_parse import urlencode +import pytest_httpbin.certs + +# Internal imports +import vcr + +from assertions import assert_cassette_has_one_response + + +def urlopen_with_cafile(*args, **kwargs): + context = ssl.create_default_context(cafile=pytest_httpbin.certs.where()) + context.check_hostname = False + kwargs["context"] = context + try: + return urlopen(*args, **kwargs) + except TypeError: + # python2/pypi don't let us override this + del kwargs["cafile"] + return urlopen(*args, **kwargs) + + +def test_response_code(httpbin_both, tmpdir): + """Ensure we can read a response code from a fetch""" + url = httpbin_both.url + with vcr.use_cassette(str(tmpdir.join("atts.yaml"))): + code = urlopen_with_cafile(url).getcode() + + with vcr.use_cassette(str(tmpdir.join("atts.yaml"))): + assert code == urlopen_with_cafile(url).getcode() + + +def test_random_body(httpbin_both, tmpdir): + """Ensure we can read the content, and that it's served from cache""" + url = httpbin_both.url + "/bytes/1024" + with vcr.use_cassette(str(tmpdir.join("body.yaml"))): + body = urlopen_with_cafile(url).read() + + with vcr.use_cassette(str(tmpdir.join("body.yaml"))): + assert body == urlopen_with_cafile(url).read() + + +def test_response_headers(httpbin_both, tmpdir): + """Ensure we can get information from the response""" + url = httpbin_both.url + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + open1 = urlopen_with_cafile(url).info().items() + + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + open2 = urlopen_with_cafile(url).info().items() + + assert sorted(open1) == sorted(open2) + + +def test_effective_url(httpbin_both, tmpdir): + """Ensure that the effective_url is captured""" + url = httpbin_both.url + "/redirect-to?url=/html" + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + effective_url = urlopen_with_cafile(url).geturl() + assert effective_url == httpbin_both.url + "/html" + + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + assert effective_url == urlopen_with_cafile(url).geturl() + + +def test_multiple_requests(httpbin_both, tmpdir): + """Ensure that we can cache multiple requests""" + urls = [httpbin_both.url, httpbin_both.url, httpbin_both.url + "/get", httpbin_both.url + "/bytes/1024"] + with vcr.use_cassette(str(tmpdir.join("multiple.yaml"))) as cass: + [urlopen_with_cafile(url) for url in urls] + assert len(cass) == len(urls) + + +def test_get_data(httpbin_both, tmpdir): + """Ensure that it works with query data""" + data = urlencode({"some": 1, "data": "here"}) + url = httpbin_both.url + "/get?" + data + with vcr.use_cassette(str(tmpdir.join("get_data.yaml"))): + res1 = urlopen_with_cafile(url).read() + + with vcr.use_cassette(str(tmpdir.join("get_data.yaml"))): + res2 = urlopen_with_cafile(url).read() + assert res1 == res2 + + +def test_post_data(httpbin_both, tmpdir): + """Ensure that it works when posting data""" + data = urlencode({"some": 1, "data": "here"}).encode("utf-8") + url = httpbin_both.url + "/post" + with vcr.use_cassette(str(tmpdir.join("post_data.yaml"))): + res1 = urlopen_with_cafile(url, data).read() + + with vcr.use_cassette(str(tmpdir.join("post_data.yaml"))) as cass: + res2 = urlopen_with_cafile(url, data).read() + assert len(cass) == 1 + + assert res1 == res2 + assert_cassette_has_one_response(cass) + + +def test_post_unicode_data(httpbin_both, tmpdir): + """Ensure that it works when posting unicode data""" + data = urlencode({"snowman": u"☃".encode("utf-8")}).encode("utf-8") + url = httpbin_both.url + "/post" + with vcr.use_cassette(str(tmpdir.join("post_data.yaml"))): + res1 = urlopen_with_cafile(url, data).read() + + with vcr.use_cassette(str(tmpdir.join("post_data.yaml"))) as cass: + res2 = urlopen_with_cafile(url, data).read() + assert len(cass) == 1 + + assert res1 == res2 + assert_cassette_has_one_response(cass) + + +def test_cross_scheme(tmpdir, httpbin_secure, httpbin): + """Ensure that requests between schemes are treated separately""" + # First fetch a url under https, and then again under https and then + # ensure that we haven't served anything out of cache, and we have two + # requests / response pairs in the cassette + with vcr.use_cassette(str(tmpdir.join("cross_scheme.yaml"))) as cass: + urlopen_with_cafile(httpbin_secure.url) + urlopen_with_cafile(httpbin.url) + assert len(cass) == 2 + assert cass.play_count == 0 + + +def test_decorator(httpbin_both, tmpdir): + """Test the decorator version of VCR.py""" + url = httpbin_both.url + + @vcr.use_cassette(str(tmpdir.join("atts.yaml"))) + def inner1(): + return urlopen_with_cafile(url).getcode() + + @vcr.use_cassette(str(tmpdir.join("atts.yaml"))) + def inner2(): + return urlopen_with_cafile(url).getcode() + + assert inner1() == inner2() diff --git a/tools/vcrpy/tests/integration/test_urllib3.py b/tools/vcrpy/tests/integration/test_urllib3.py new file mode 100644 index 000000000000..110c05af2961 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_urllib3.py @@ -0,0 +1,159 @@ +"""Integration tests with urllib3""" + +# coding=utf-8 + +import pytest +import pytest_httpbin +import vcr +from vcr.patch import force_reset +from assertions import assert_cassette_empty, assert_is_json + +urllib3 = pytest.importorskip("urllib3") + + +@pytest.fixture(scope="module") +def verify_pool_mgr(): + return urllib3.PoolManager( + cert_reqs="CERT_REQUIRED", ca_certs=pytest_httpbin.certs.where() # Force certificate check. + ) + + +@pytest.fixture(scope="module") +def pool_mgr(): + return urllib3.PoolManager(cert_reqs="CERT_NONE") + + +def test_status_code(httpbin_both, tmpdir, verify_pool_mgr): + """Ensure that we can read the status code""" + url = httpbin_both.url + with vcr.use_cassette(str(tmpdir.join("atts.yaml"))): + status_code = verify_pool_mgr.request("GET", url).status + + with vcr.use_cassette(str(tmpdir.join("atts.yaml"))): + assert status_code == verify_pool_mgr.request("GET", url).status + + +def test_headers(tmpdir, httpbin_both, verify_pool_mgr): + """Ensure that we can read the headers back""" + url = httpbin_both.url + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + headers = verify_pool_mgr.request("GET", url).headers + + with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): + assert headers == verify_pool_mgr.request("GET", url).headers + + +def test_body(tmpdir, httpbin_both, verify_pool_mgr): + """Ensure the responses are all identical enough""" + url = httpbin_both.url + "/bytes/1024" + with vcr.use_cassette(str(tmpdir.join("body.yaml"))): + content = verify_pool_mgr.request("GET", url).data + + with vcr.use_cassette(str(tmpdir.join("body.yaml"))): + assert content == verify_pool_mgr.request("GET", url).data + + +def test_auth(tmpdir, httpbin_both, verify_pool_mgr): + """Ensure that we can handle basic auth""" + auth = ("user", "passwd") + headers = urllib3.util.make_headers(basic_auth="{}:{}".format(*auth)) + url = httpbin_both.url + "/basic-auth/user/passwd" + with vcr.use_cassette(str(tmpdir.join("auth.yaml"))): + one = verify_pool_mgr.request("GET", url, headers=headers) + + with vcr.use_cassette(str(tmpdir.join("auth.yaml"))): + two = verify_pool_mgr.request("GET", url, headers=headers) + assert one.data == two.data + assert one.status == two.status + + +def test_auth_failed(tmpdir, httpbin_both, verify_pool_mgr): + """Ensure that we can save failed auth statuses""" + auth = ("user", "wrongwrongwrong") + headers = urllib3.util.make_headers(basic_auth="{}:{}".format(*auth)) + url = httpbin_both.url + "/basic-auth/user/passwd" + with vcr.use_cassette(str(tmpdir.join("auth-failed.yaml"))) as cass: + # Ensure that this is empty to begin with + assert_cassette_empty(cass) + one = verify_pool_mgr.request("GET", url, headers=headers) + two = verify_pool_mgr.request("GET", url, headers=headers) + assert one.data == two.data + assert one.status == two.status == 401 + + +def test_post(tmpdir, httpbin_both, verify_pool_mgr): + """Ensure that we can post and cache the results""" + data = {"key1": "value1", "key2": "value2"} + url = httpbin_both.url + "/post" + with vcr.use_cassette(str(tmpdir.join("verify_pool_mgr.yaml"))): + req1 = verify_pool_mgr.request("POST", url, data).data + + with vcr.use_cassette(str(tmpdir.join("verify_pool_mgr.yaml"))): + req2 = verify_pool_mgr.request("POST", url, data).data + + assert req1 == req2 + + +def test_redirects(tmpdir, httpbin_both, verify_pool_mgr): + """Ensure that we can handle redirects""" + url = httpbin_both.url + "/redirect-to?url=bytes/1024" + with vcr.use_cassette(str(tmpdir.join("verify_pool_mgr.yaml"))): + content = verify_pool_mgr.request("GET", url).data + + with vcr.use_cassette(str(tmpdir.join("verify_pool_mgr.yaml"))) as cass: + assert content == verify_pool_mgr.request("GET", url).data + # Ensure that we've now cached *two* responses. One for the redirect + # and one for the final fetch + assert len(cass) == 2 + assert cass.play_count == 2 + + +def test_cross_scheme(tmpdir, httpbin, httpbin_secure, verify_pool_mgr): + """Ensure that requests between schemes are treated separately""" + # First fetch a url under http, and then again under https and then + # ensure that we haven't served anything out of cache, and we have two + # requests / response pairs in the cassette + with vcr.use_cassette(str(tmpdir.join("cross_scheme.yaml"))) as cass: + verify_pool_mgr.request("GET", httpbin_secure.url) + verify_pool_mgr.request("GET", httpbin.url) + assert cass.play_count == 0 + assert len(cass) == 2 + + +def test_gzip(tmpdir, httpbin_both, verify_pool_mgr): + """ + Ensure that requests (actually urllib3) is able to automatically decompress + the response body + """ + url = httpbin_both.url + "/gzip" + response = verify_pool_mgr.request("GET", url) + + with vcr.use_cassette(str(tmpdir.join("gzip.yaml"))): + response = verify_pool_mgr.request("GET", url) + assert_is_json(response.data) + + with vcr.use_cassette(str(tmpdir.join("gzip.yaml"))): + assert_is_json(response.data) + + +def test_https_with_cert_validation_disabled(tmpdir, httpbin_secure, pool_mgr): + with vcr.use_cassette(str(tmpdir.join("cert_validation_disabled.yaml"))): + pool_mgr.request("GET", httpbin_secure.url) + + +def test_urllib3_force_reset(): + cpool = urllib3.connectionpool + http_original = cpool.HTTPConnection + https_original = cpool.HTTPSConnection + verified_https_original = cpool.VerifiedHTTPSConnection + with vcr.use_cassette(path="test"): + first_cassette_HTTPConnection = cpool.HTTPConnection + first_cassette_HTTPSConnection = cpool.HTTPSConnection + first_cassette_VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection + with force_reset(): + assert cpool.HTTPConnection is http_original + assert cpool.HTTPSConnection is https_original + assert cpool.VerifiedHTTPSConnection is verified_https_original + assert cpool.HTTPConnection is first_cassette_HTTPConnection + assert cpool.HTTPSConnection is first_cassette_HTTPSConnection + assert cpool.VerifiedHTTPSConnection is first_cassette_VerifiedHTTPSConnection diff --git a/tools/vcrpy/tests/integration/test_wild.py b/tools/vcrpy/tests/integration/test_wild.py new file mode 100644 index 000000000000..7fe57d7dd3b2 --- /dev/null +++ b/tools/vcrpy/tests/integration/test_wild.py @@ -0,0 +1,109 @@ +import multiprocessing +import pytest +from six.moves import xmlrpc_client, xmlrpc_server + +requests = pytest.importorskip("requests") + +import vcr # NOQA + +try: + import httplib +except ImportError: + import http.client as httplib + + +def test_domain_redirect(): + """Ensure that redirects across domains are considered unique""" + # In this example, seomoz.org redirects to moz.com, and if those + # requests are considered identical, then we'll be stuck in a redirect + # loop. + url = "http://seomoz.org/" + with vcr.use_cassette("tests/fixtures/wild/domain_redirect.yaml") as cass: + requests.get(url, headers={"User-Agent": "vcrpy-test"}) + # Ensure that we've now served two responses. One for the original + # redirect, and a second for the actual fetch + assert len(cass) == 2 + + +def test_flickr_multipart_upload(httpbin, tmpdir): + """ + The python-flickr-api project does a multipart + upload that confuses vcrpy + """ + + def _pretend_to_be_flickr_library(): + content_type, body = "text/plain", "HELLO WORLD" + h = httplib.HTTPConnection(httpbin.host, httpbin.port) + headers = {"Content-Type": content_type, "content-length": str(len(body))} + h.request("POST", "/post/", headers=headers) + h.send(body) + r = h.getresponse() + data = r.read() + h.close() + + return data + + testfile = str(tmpdir.join("flickr.yml")) + with vcr.use_cassette(testfile) as cass: + _pretend_to_be_flickr_library() + assert len(cass) == 1 + + with vcr.use_cassette(testfile) as cass: + assert len(cass) == 1 + _pretend_to_be_flickr_library() + assert cass.play_count == 1 + + +def test_flickr_should_respond_with_200(tmpdir): + testfile = str(tmpdir.join("flickr.yml")) + with vcr.use_cassette(testfile): + r = requests.post("https://api.flickr.com/services/upload", verify=False) + assert r.status_code == 200 + + +def test_cookies(tmpdir, httpbin): + testfile = str(tmpdir.join("cookies.yml")) + with vcr.use_cassette(testfile): + s = requests.Session() + s.get(httpbin.url + "/cookies/set?k1=v1&k2=v2") + + r2 = s.get(httpbin.url + "/cookies") + assert len(r2.json()["cookies"]) == 2 + + +def test_amazon_doctype(tmpdir): + # amazon gzips its homepage. For some reason, in requests 2.7, it's not + # getting gunzipped. + with vcr.use_cassette(str(tmpdir.join("amz.yml"))): + r = requests.get("http://www.amazon.com", verify=False) + assert "html" in r.text + + +def start_rpc_server(q): + httpd = xmlrpc_server.SimpleXMLRPCServer(("127.0.0.1", 0)) + httpd.register_function(pow) + q.put("http://{}:{}".format(*httpd.server_address)) + httpd.serve_forever() + + +@pytest.yield_fixture(scope="session") +def rpc_server(): + q = multiprocessing.Queue() + proxy_process = multiprocessing.Process(target=start_rpc_server, args=(q,)) + try: + proxy_process.start() + yield q.get() + finally: + proxy_process.terminate() + + +def test_xmlrpclib(tmpdir, rpc_server): + with vcr.use_cassette(str(tmpdir.join("xmlrpcvideo.yaml"))): + roundup_server = xmlrpc_client.ServerProxy(rpc_server, allow_none=True) + original_schema = roundup_server.pow(2, 4) + + with vcr.use_cassette(str(tmpdir.join("xmlrpcvideo.yaml"))): + roundup_server = xmlrpc_client.ServerProxy(rpc_server, allow_none=True) + second_schema = roundup_server.pow(2, 4) + + assert original_schema == second_schema diff --git a/tools/vcrpy/tests/unit/test_cassettes.py b/tools/vcrpy/tests/unit/test_cassettes.py new file mode 100644 index 000000000000..3e35d5ac2332 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_cassettes.py @@ -0,0 +1,370 @@ +import copy +import inspect +import os + +from six.moves import http_client as httplib +import pytest +import yaml + +from vcr.compat import mock, contextlib +from vcr.cassette import Cassette +from vcr.errors import UnhandledHTTPRequestError +from vcr.patch import force_reset +from vcr.stubs import VCRHTTPSConnection + + +def test_cassette_load(tmpdir): + a_file = tmpdir.join("test_cassette.yml") + a_file.write( + yaml.dump( + { + "interactions": [ + {"request": {"body": "", "uri": "foo", "method": "GET", "headers": {}}, "response": "bar"} + ] + } + ) + ) + a_cassette = Cassette.load(path=str(a_file)) + assert len(a_cassette) == 1 + + +def test_cassette_not_played(): + a = Cassette("test") + assert not a.play_count + + +def test_cassette_append(): + a = Cassette("test") + a.append("foo", "bar") + assert a.requests == ["foo"] + assert a.responses == ["bar"] + + +def test_cassette_len(): + a = Cassette("test") + a.append("foo", "bar") + a.append("foo2", "bar2") + assert len(a) == 2 + + +def _mock_requests_match(request1, request2, matchers): + return request1 == request2 + + +@mock.patch("vcr.cassette.requests_match", _mock_requests_match) +def test_cassette_contains(): + a = Cassette("test") + a.append("foo", "bar") + assert "foo" in a + + +@mock.patch("vcr.cassette.requests_match", _mock_requests_match) +def test_cassette_responses_of(): + a = Cassette("test") + a.append("foo", "bar") + assert a.responses_of("foo") == ["bar"] + + +@mock.patch("vcr.cassette.requests_match", _mock_requests_match) +def test_cassette_get_missing_response(): + a = Cassette("test") + with pytest.raises(UnhandledHTTPRequestError): + a.responses_of("foo") + + +@mock.patch("vcr.cassette.requests_match", _mock_requests_match) +def test_cassette_cant_read_same_request_twice(): + a = Cassette("test") + a.append("foo", "bar") + a.play_response("foo") + with pytest.raises(UnhandledHTTPRequestError): + a.play_response("foo") + + +def make_get_request(): + conn = httplib.HTTPConnection("www.python.org") + conn.request("GET", "/index.html") + return conn.getresponse() + + +@mock.patch("vcr.cassette.requests_match", return_value=True) +@mock.patch( + "vcr.cassette.FilesystemPersister.load_cassette", + classmethod(lambda *args, **kwargs: (("foo",), (mock.MagicMock(),))), +) +@mock.patch("vcr.cassette.Cassette.can_play_response_for", return_value=True) +@mock.patch("vcr.stubs.VCRHTTPResponse") +def test_function_decorated_with_use_cassette_can_be_invoked_multiple_times(*args): + decorated_function = Cassette.use(path="test")(make_get_request) + for i in range(4): + decorated_function() + + +def test_arg_getter_functionality(): + arg_getter = mock.Mock(return_value={"path": "test"}) + context_decorator = Cassette.use_arg_getter(arg_getter) + + with context_decorator as cassette: + assert cassette._path == "test" + + arg_getter.return_value = {"path": "other"} + + with context_decorator as cassette: + assert cassette._path == "other" + + arg_getter.return_value = {"path": "other", "filter_headers": ("header_name",)} + + @context_decorator + def function(): + pass + + with mock.patch.object(Cassette, "load", return_value=mock.MagicMock(inject=False)) as cassette_load: + function() + cassette_load.assert_called_once_with(**arg_getter.return_value) + + +def test_cassette_not_all_played(): + a = Cassette("test") + a.append("foo", "bar") + assert not a.all_played + + +@mock.patch("vcr.cassette.requests_match", _mock_requests_match) +def test_cassette_all_played(): + a = Cassette("test") + a.append("foo", "bar") + a.play_response("foo") + assert a.all_played + + +@mock.patch("vcr.cassette.requests_match", _mock_requests_match) +def test_cassette_rewound(): + a = Cassette("test") + a.append("foo", "bar") + a.play_response("foo") + assert a.all_played + + a.rewind() + assert not a.all_played + + +def test_before_record_response(): + before_record_response = mock.Mock(return_value="mutated") + cassette = Cassette("test", before_record_response=before_record_response) + cassette.append("req", "res") + + before_record_response.assert_called_once_with("res") + assert cassette.responses[0] == "mutated" + + +def assert_get_response_body_is(value): + conn = httplib.HTTPConnection("www.python.org") + conn.request("GET", "/index.html") + assert conn.getresponse().read().decode("utf8") == value + + +@mock.patch("vcr.cassette.requests_match", _mock_requests_match) +@mock.patch("vcr.cassette.Cassette.can_play_response_for", return_value=True) +@mock.patch("vcr.cassette.Cassette._save", return_value=True) +def test_nesting_cassette_context_managers(*args): + first_response = { + "body": {"string": b"first_response"}, + "headers": {}, + "status": {"message": "m", "code": 200}, + } + + second_response = copy.deepcopy(first_response) + second_response["body"]["string"] = b"second_response" + + with contextlib.ExitStack() as exit_stack: + first_cassette = exit_stack.enter_context(Cassette.use(path="test")) + exit_stack.enter_context( + mock.patch.object(first_cassette, "play_response", return_value=first_response) + ) + assert_get_response_body_is("first_response") + + # Make sure a second cassette can supercede the first + with Cassette.use(path="test") as second_cassette: + with mock.patch.object(second_cassette, "play_response", return_value=second_response): + assert_get_response_body_is("second_response") + + # Now the first cassette should be back in effect + assert_get_response_body_is("first_response") + + +def test_nesting_context_managers_by_checking_references_of_http_connection(): + original = httplib.HTTPConnection + with Cassette.use(path="test"): + first_cassette_HTTPConnection = httplib.HTTPConnection + with Cassette.use(path="test"): + second_cassette_HTTPConnection = httplib.HTTPConnection + assert second_cassette_HTTPConnection is not first_cassette_HTTPConnection + with Cassette.use(path="test"): + assert httplib.HTTPConnection is not second_cassette_HTTPConnection + with force_reset(): + assert httplib.HTTPConnection is original + assert httplib.HTTPConnection is second_cassette_HTTPConnection + assert httplib.HTTPConnection is first_cassette_HTTPConnection + + +def test_custom_patchers(): + class Test(object): + attribute = None + + with Cassette.use(path="custom_patches", custom_patches=((Test, "attribute", VCRHTTPSConnection),)): + assert issubclass(Test.attribute, VCRHTTPSConnection) + assert VCRHTTPSConnection is not Test.attribute + old_attribute = Test.attribute + + with Cassette.use(path="custom_patches", custom_patches=((Test, "attribute", VCRHTTPSConnection),)): + assert issubclass(Test.attribute, VCRHTTPSConnection) + assert VCRHTTPSConnection is not Test.attribute + assert Test.attribute is not old_attribute + + assert issubclass(Test.attribute, VCRHTTPSConnection) + assert VCRHTTPSConnection is not Test.attribute + assert Test.attribute is old_attribute + + +def test_decorated_functions_are_reentrant(): + info = {"second": False} + original_conn = httplib.HTTPConnection + + @Cassette.use(path="whatever", inject=True) + def test_function(cassette): + if info["second"]: + assert httplib.HTTPConnection is not info["first_conn"] + else: + info["first_conn"] = httplib.HTTPConnection + info["second"] = True + test_function() + assert httplib.HTTPConnection is info["first_conn"] + + test_function() + assert httplib.HTTPConnection is original_conn + + +def test_cassette_use_called_without_path_uses_function_to_generate_path(): + @Cassette.use(inject=True) + def function_name(cassette): + assert cassette._path == "function_name" + + function_name() + + +def test_path_transformer_with_function_path(): + def path_transformer(path): + return os.path.join("a", path) + + @Cassette.use(inject=True, path_transformer=path_transformer) + def function_name(cassette): + assert cassette._path == os.path.join("a", "function_name") + + function_name() + + +def test_path_transformer_with_context_manager(): + with Cassette.use(path="b", path_transformer=lambda *args: "a") as cassette: + assert cassette._path == "a" + + +def test_path_transformer_None(): + with Cassette.use(path="a", path_transformer=None) as cassette: + assert cassette._path == "a" + + +def test_func_path_generator(): + def generator(function): + return os.path.join(os.path.dirname(inspect.getfile(function)), function.__name__) + + @Cassette.use(inject=True, func_path_generator=generator) + def function_name(cassette): + assert cassette._path == os.path.join(os.path.dirname(__file__), "function_name") + + function_name() + + +def test_use_as_decorator_on_coroutine(): + original_http_connetion = httplib.HTTPConnection + + @Cassette.use(inject=True) + def test_function(cassette): + assert httplib.HTTPConnection.cassette is cassette + assert httplib.HTTPConnection is not original_http_connetion + value = yield 1 + assert value == 1 + assert httplib.HTTPConnection.cassette is cassette + assert httplib.HTTPConnection is not original_http_connetion + value = yield 2 + assert value == 2 + + coroutine = test_function() + value = next(coroutine) + while True: + try: + value = coroutine.send(value) + except StopIteration: + break + + +def test_use_as_decorator_on_generator(): + original_http_connetion = httplib.HTTPConnection + + @Cassette.use(inject=True) + def test_function(cassette): + assert httplib.HTTPConnection.cassette is cassette + assert httplib.HTTPConnection is not original_http_connetion + yield 1 + assert httplib.HTTPConnection.cassette is cassette + assert httplib.HTTPConnection is not original_http_connetion + yield 2 + + assert list(test_function()) == [1, 2] + + +@mock.patch("vcr.cassette.get_matchers_results") +def test_find_requests_with_most_matches_one_similar_request(mock_get_matchers_results): + mock_get_matchers_results.side_effect = [ + (["method"], [("path", "failed : path"), ("query", "failed : query")]), + (["method", "path"], [("query", "failed : query")]), + ([], [("method", "failed : method"), ("path", "failed : path"), ("query", "failed : query")]), + ] + + cassette = Cassette("test") + for request in range(1, 4): + cassette.append(request, "response") + result = cassette.find_requests_with_most_matches("fake request") + assert result == [(2, ["method", "path"], [("query", "failed : query")])] + + +@mock.patch("vcr.cassette.get_matchers_results") +def test_find_requests_with_most_matches_no_similar_requests(mock_get_matchers_results): + mock_get_matchers_results.side_effect = [ + ([], [("path", "failed : path"), ("query", "failed : query")]), + ([], [("path", "failed : path"), ("query", "failed : query")]), + ([], [("path", "failed : path"), ("query", "failed : query")]), + ] + + cassette = Cassette("test") + for request in range(1, 4): + cassette.append(request, "response") + result = cassette.find_requests_with_most_matches("fake request") + assert result == [] + + +@mock.patch("vcr.cassette.get_matchers_results") +def test_find_requests_with_most_matches_many_similar_requests(mock_get_matchers_results): + mock_get_matchers_results.side_effect = [ + (["method", "path"], [("query", "failed : query")]), + (["method"], [("path", "failed : path"), ("query", "failed : query")]), + (["method", "path"], [("query", "failed : query")]), + ] + + cassette = Cassette("test") + for request in range(1, 4): + cassette.append(request, "response") + result = cassette.find_requests_with_most_matches("fake request") + assert result == [ + (1, ["method", "path"], [("query", "failed : query")]), + (3, ["method", "path"], [("query", "failed : query")]), + ] diff --git a/tools/vcrpy/tests/unit/test_errors.py b/tools/vcrpy/tests/unit/test_errors.py new file mode 100644 index 000000000000..fb2ab806b174 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_errors.py @@ -0,0 +1,68 @@ +import pytest + +from vcr.compat import mock +from vcr import errors +from vcr.cassette import Cassette + + +@mock.patch("vcr.cassette.Cassette.find_requests_with_most_matches") +@pytest.mark.parametrize( + "most_matches, expected_message", + [ + # No request match found + ([], "No similar requests, that have not been played, found."), + # One matcher failed + ( + [("similar request", ["method", "path"], [("query", "failed : query")])], + "Found 1 similar requests with 1 different matcher(s) :\n" + "\n1 - ('similar request').\n" + "Matchers succeeded : ['method', 'path']\n" + "Matchers failed :\n" + "query - assertion failure :\n" + "failed : query\n", + ), + # Multiple failed matchers + ( + [("similar request", ["method"], [("query", "failed : query"), ("path", "failed : path")])], + "Found 1 similar requests with 2 different matcher(s) :\n" + "\n1 - ('similar request').\n" + "Matchers succeeded : ['method']\n" + "Matchers failed :\n" + "query - assertion failure :\n" + "failed : query\n" + "path - assertion failure :\n" + "failed : path\n", + ), + # Multiple similar requests + ( + [ + ("similar request", ["method"], [("query", "failed : query")]), + ("similar request 2", ["method"], [("query", "failed : query 2")]), + ], + "Found 2 similar requests with 1 different matcher(s) :\n" + "\n1 - ('similar request').\n" + "Matchers succeeded : ['method']\n" + "Matchers failed :\n" + "query - assertion failure :\n" + "failed : query\n" + "\n2 - ('similar request 2').\n" + "Matchers succeeded : ['method']\n" + "Matchers failed :\n" + "query - assertion failure :\n" + "failed : query 2\n", + ), + ], +) +def test_CannotOverwriteExistingCassetteException_get_message( + mock_find_requests_with_most_matches, most_matches, expected_message +): + mock_find_requests_with_most_matches.return_value = most_matches + cassette = Cassette("path") + failed_request = "request" + exception_message = errors.CannotOverwriteExistingCassetteException._get_message(cassette, "request") + expected = ( + "Can't overwrite existing cassette (%r) in your current record mode (%r).\n" + "No match for the request (%r) was found.\n" + "%s" % (cassette._path, cassette.record_mode, failed_request, expected_message) + ) + assert exception_message == expected diff --git a/tools/vcrpy/tests/unit/test_filters.py b/tools/vcrpy/tests/unit/test_filters.py new file mode 100644 index 000000000000..bfdde2038976 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_filters.py @@ -0,0 +1,283 @@ +from six import BytesIO +from vcr.filters import ( + remove_headers, + replace_headers, + remove_query_parameters, + replace_query_parameters, + remove_post_data_parameters, + replace_post_data_parameters, + decode_response, +) +from vcr.compat import mock +from vcr.request import Request +import gzip +import json +import zlib + + +def test_replace_headers(): + # This tests all of: + # 1. keeping a header + # 2. removing a header + # 3. replacing a header + # 4. replacing a header using a callable + # 5. removing a header using a callable + # 6. replacing a header that doesn't exist + headers = {"one": ["keep"], "two": ["lose"], "three": ["change"], "four": ["shout"], "five": ["whisper"]} + request = Request("GET", "http://google.com", "", headers) + replace_headers( + request, + [ + ("two", None), + ("three", "tada"), + ("four", lambda key, value, request: value.upper()), + ("five", lambda key, value, request: None), + ("six", "doesntexist"), + ], + ) + assert request.headers == {"one": "keep", "three": "tada", "four": "SHOUT"} + + +def test_replace_headers_empty(): + headers = {"hello": "goodbye", "secret": "header"} + request = Request("GET", "http://google.com", "", headers) + replace_headers(request, []) + assert request.headers == headers + + +def test_replace_headers_callable(): + # This goes beyond test_replace_headers() to ensure that the callable + # receives the expected arguments. + headers = {"hey": "there"} + request = Request("GET", "http://google.com", "", headers) + callme = mock.Mock(return_value="ho") + replace_headers(request, [("hey", callme)]) + assert request.headers == {"hey": "ho"} + assert callme.call_args == ((), {"request": request, "key": "hey", "value": "there"}) + + +def test_remove_headers(): + # Test the backward-compatible API wrapper. + headers = {"hello": ["goodbye"], "secret": ["header"]} + request = Request("GET", "http://google.com", "", headers) + remove_headers(request, ["secret"]) + assert request.headers == {"hello": "goodbye"} + + +def test_replace_query_parameters(): + # This tests all of: + # 1. keeping a parameter + # 2. removing a parameter + # 3. replacing a parameter + # 4. replacing a parameter using a callable + # 5. removing a parameter using a callable + # 6. replacing a parameter that doesn't exist + uri = "http://g.com/?one=keep&two=lose&three=change&four=shout&five=whisper" + request = Request("GET", uri, "", {}) + replace_query_parameters( + request, + [ + ("two", None), + ("three", "tada"), + ("four", lambda key, value, request: value.upper()), + ("five", lambda key, value, request: None), + ("six", "doesntexist"), + ], + ) + assert request.query == [("four", "SHOUT"), ("one", "keep"), ("three", "tada")] + + +def test_remove_all_query_parameters(): + uri = "http://g.com/?q=cowboys&w=1" + request = Request("GET", uri, "", {}) + replace_query_parameters(request, [("w", None), ("q", None)]) + assert request.uri == "http://g.com/" + + +def test_replace_query_parameters_callable(): + # This goes beyond test_replace_query_parameters() to ensure that the + # callable receives the expected arguments. + uri = "http://g.com/?hey=there" + request = Request("GET", uri, "", {}) + callme = mock.Mock(return_value="ho") + replace_query_parameters(request, [("hey", callme)]) + assert request.uri == "http://g.com/?hey=ho" + assert callme.call_args == ((), {"request": request, "key": "hey", "value": "there"}) + + +def test_remove_query_parameters(): + # Test the backward-compatible API wrapper. + uri = "http://g.com/?q=cowboys&w=1" + request = Request("GET", uri, "", {}) + remove_query_parameters(request, ["w"]) + assert request.uri == "http://g.com/?q=cowboys" + + +def test_replace_post_data_parameters(): + # This tests all of: + # 1. keeping a parameter + # 2. removing a parameter + # 3. replacing a parameter + # 4. replacing a parameter using a callable + # 5. removing a parameter using a callable + # 6. replacing a parameter that doesn't exist + body = b"one=keep&two=lose&three=change&four=shout&five=whisper" + request = Request("POST", "http://google.com", body, {}) + replace_post_data_parameters( + request, + [ + ("two", None), + ("three", "tada"), + ("four", lambda key, value, request: value.upper()), + ("five", lambda key, value, request: None), + ("six", "doesntexist"), + ], + ) + assert request.body == b"one=keep&three=tada&four=SHOUT" + + +def test_replace_post_data_parameters_empty_body(): + # This test ensures replace_post_data_parameters doesn't throw exception when body is empty. + body = None + request = Request("POST", "http://google.com", body, {}) + replace_post_data_parameters( + request, + [ + ("two", None), + ("three", "tada"), + ("four", lambda key, value, request: value.upper()), + ("five", lambda key, value, request: None), + ("six", "doesntexist"), + ], + ) + assert request.body is None + + +def test_remove_post_data_parameters(): + # Test the backward-compatible API wrapper. + body = b"id=secret&foo=bar" + request = Request("POST", "http://google.com", body, {}) + remove_post_data_parameters(request, ["id"]) + assert request.body == b"foo=bar" + + +def test_preserve_multiple_post_data_parameters(): + body = b"id=secret&foo=bar&foo=baz" + request = Request("POST", "http://google.com", body, {}) + replace_post_data_parameters(request, [("id", None)]) + assert request.body == b"foo=bar&foo=baz" + + +def test_remove_all_post_data_parameters(): + body = b"id=secret&foo=bar" + request = Request("POST", "http://google.com", body, {}) + replace_post_data_parameters(request, [("id", None), ("foo", None)]) + assert request.body == b"" + + +def test_replace_json_post_data_parameters(): + # This tests all of: + # 1. keeping a parameter + # 2. removing a parameter + # 3. replacing a parameter + # 4. replacing a parameter using a callable + # 5. removing a parameter using a callable + # 6. replacing a parameter that doesn't exist + body = b'{"one": "keep", "two": "lose", "three": "change", "four": "shout", "five": "whisper"}' + request = Request("POST", "http://google.com", body, {}) + request.headers["Content-Type"] = "application/json" + replace_post_data_parameters( + request, + [ + ("two", None), + ("three", "tada"), + ("four", lambda key, value, request: value.upper()), + ("five", lambda key, value, request: None), + ("six", "doesntexist"), + ], + ) + request_data = json.loads(request.body.decode("utf-8")) + expected_data = json.loads('{"one": "keep", "three": "tada", "four": "SHOUT"}') + assert request_data == expected_data + + +def test_remove_json_post_data_parameters(): + # Test the backward-compatible API wrapper. + body = b'{"id": "secret", "foo": "bar", "baz": "qux"}' + request = Request("POST", "http://google.com", body, {}) + request.headers["Content-Type"] = "application/json" + remove_post_data_parameters(request, ["id"]) + request_body_json = json.loads(request.body.decode("utf-8")) + expected_json = json.loads(b'{"foo": "bar", "baz": "qux"}'.decode("utf-8")) + assert request_body_json == expected_json + + +def test_remove_all_json_post_data_parameters(): + body = b'{"id": "secret", "foo": "bar"}' + request = Request("POST", "http://google.com", body, {}) + request.headers["Content-Type"] = "application/json" + replace_post_data_parameters(request, [("id", None), ("foo", None)]) + assert request.body == b"{}" + + +def test_decode_response_uncompressed(): + recorded_response = { + "status": {"message": "OK", "code": 200}, + "headers": { + "content-length": ["10806"], + "date": ["Fri, 24 Oct 2014 18:35:37 GMT"], + "content-type": ["text/html; charset=utf-8"], + }, + "body": {"string": b""}, + } + assert decode_response(recorded_response) == recorded_response + + +def test_decode_response_deflate(): + body = b"deflate message" + deflate_response = { + "body": {"string": zlib.compress(body)}, + "headers": { + "access-control-allow-credentials": ["true"], + "access-control-allow-origin": ["*"], + "connection": ["keep-alive"], + "content-encoding": ["deflate"], + "content-length": ["177"], + "content-type": ["application/json"], + "date": ["Wed, 02 Dec 2015 19:44:32 GMT"], + "server": ["nginx"], + }, + "status": {"code": 200, "message": "OK"}, + } + decoded_response = decode_response(deflate_response) + assert decoded_response["body"]["string"] == body + assert decoded_response["headers"]["content-length"] == [str(len(body))] + + +def test_decode_response_gzip(): + body = b"gzip message" + + buf = BytesIO() + f = gzip.GzipFile("a", fileobj=buf, mode="wb") + f.write(body) + f.close() + + compressed_body = buf.getvalue() + buf.close() + gzip_response = { + "body": {"string": compressed_body}, + "headers": { + "access-control-allow-credentials": ["true"], + "access-control-allow-origin": ["*"], + "connection": ["keep-alive"], + "content-encoding": ["gzip"], + "content-length": ["177"], + "content-type": ["application/json"], + "date": ["Wed, 02 Dec 2015 19:44:32 GMT"], + "server": ["nginx"], + }, + "status": {"code": 200, "message": "OK"}, + } + decoded_response = decode_response(gzip_response) + assert decoded_response["body"]["string"] == body + assert decoded_response["headers"]["content-length"] == [str(len(body))] diff --git a/tools/vcrpy/tests/unit/test_json_serializer.py b/tools/vcrpy/tests/unit/test_json_serializer.py new file mode 100644 index 000000000000..880c36ac0438 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_json_serializer.py @@ -0,0 +1,17 @@ +import pytest +from vcr.serializers.jsonserializer import serialize +from vcr.request import Request + + +def test_serialize_binary(): + request = Request(method="GET", uri="http://localhost/", body="", headers={}) + cassette = {"requests": [request], "responses": [{"body": b"\x8c"}]} + + with pytest.raises(Exception) as e: + serialize(cassette) + assert ( + e.message + == "Error serializing cassette to JSON. Does this \ + HTTP interaction contain binary data? If so, use a different \ + serializer (like the yaml serializer) for this request" + ) diff --git a/tools/vcrpy/tests/unit/test_matchers.py b/tools/vcrpy/tests/unit/test_matchers.py new file mode 100644 index 000000000000..34eadd350610 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_matchers.py @@ -0,0 +1,274 @@ +import itertools +from vcr.compat import mock + +import pytest + +from vcr import matchers +from vcr import request + +# the dict contains requests with corresponding to its key difference +# with 'base' request. +REQUESTS = { + "base": request.Request("GET", "http://host.com/p?a=b", "", {}), + "method": request.Request("POST", "http://host.com/p?a=b", "", {}), + "scheme": request.Request("GET", "https://host.com:80/p?a=b", "", {}), + "host": request.Request("GET", "http://another-host.com/p?a=b", "", {}), + "port": request.Request("GET", "http://host.com:90/p?a=b", "", {}), + "path": request.Request("GET", "http://host.com/x?a=b", "", {}), + "query": request.Request("GET", "http://host.com/p?c=d", "", {}), +} + + +def assert_matcher(matcher_name): + matcher = getattr(matchers, matcher_name) + for k1, k2 in itertools.permutations(REQUESTS, 2): + expecting_assertion_error = matcher_name in {k1, k2} + if expecting_assertion_error: + with pytest.raises(AssertionError): + matcher(REQUESTS[k1], REQUESTS[k2]) + else: + assert matcher(REQUESTS[k1], REQUESTS[k2]) is None + + +def test_uri_matcher(): + for k1, k2 in itertools.permutations(REQUESTS, 2): + expecting_assertion_error = {k1, k2} != {"base", "method"} + if expecting_assertion_error: + with pytest.raises(AssertionError): + matchers.uri(REQUESTS[k1], REQUESTS[k2]) + else: + assert matchers.uri(REQUESTS[k1], REQUESTS[k2]) is None + + +req1_body = ( + b"test" + b"" + b"a1" + b"b2" + b"" +) +req2_body = ( + b"test" + b"" + b"b2" + b"a1" + b"" +) +boto3_bytes_headers = { + "X-Amz-Content-SHA256": b"UNSIGNED-PAYLOAD", + "Cache-Control": b"max-age=31536000, public", + "X-Amz-Date": b"20191102T143910Z", + "User-Agent": b"Boto3/1.9.102 Python/3.5.3 Linux/4.15.0-54-generic Botocore/1.12.253 Resource", + "Content-MD5": b"GQqjEXsRqrPyxfTl99nkAg==", + "Content-Type": b"text/plain", + "Expect": b"100-continue", + "Content-Length": "21", +} + + +@pytest.mark.parametrize( + "r1, r2", + [ + ( + request.Request("POST", "http://host.com/", "123", {}), + request.Request("POST", "http://another-host.com/", "123", {"Some-Header": "value"}), + ), + ( + request.Request( + "POST", "http://host.com/", "a=1&b=2", {"Content-Type": "application/x-www-form-urlencoded"} + ), + request.Request( + "POST", "http://host.com/", "b=2&a=1", {"Content-Type": "application/x-www-form-urlencoded"} + ), + ), + ( + request.Request("POST", "http://host.com/", "123", {}), + request.Request("POST", "http://another-host.com/", "123", {"Some-Header": "value"}), + ), + ( + request.Request( + "POST", "http://host.com/", "a=1&b=2", {"Content-Type": "application/x-www-form-urlencoded"} + ), + request.Request( + "POST", "http://host.com/", "b=2&a=1", {"Content-Type": "application/x-www-form-urlencoded"} + ), + ), + ( + request.Request( + "POST", "http://host.com/", '{"a": 1, "b": 2}', {"Content-Type": "application/json"} + ), + request.Request( + "POST", "http://host.com/", '{"b": 2, "a": 1}', {"content-type": "application/json"} + ), + ), + ( + request.Request( + "POST", "http://host.com/", req1_body, {"User-Agent": "xmlrpclib", "Content-Type": "text/xml"} + ), + request.Request( + "POST", + "http://host.com/", + req2_body, + {"user-agent": "somexmlrpc", "content-type": "text/xml"}, + ), + ), + ( + request.Request( + "POST", "http://host.com/", '{"a": 1, "b": 2}', {"Content-Type": "application/json"} + ), + request.Request( + "POST", "http://host.com/", '{"b": 2, "a": 1}', {"content-type": "application/json"} + ), + ), + ( + # special case for boto3 bytes headers + request.Request("POST", "http://aws.custom.com/", b"123", boto3_bytes_headers), + request.Request("POST", "http://aws.custom.com/", b"123", boto3_bytes_headers), + ), + ], +) +def test_body_matcher_does_match(r1, r2): + assert matchers.body(r1, r2) is None + + +@pytest.mark.parametrize( + "r1, r2", + [ + ( + request.Request("POST", "http://host.com/", '{"a": 1, "b": 2}', {}), + request.Request("POST", "http://host.com/", '{"b": 2, "a": 1}', {}), + ), + ( + request.Request( + "POST", "http://host.com/", '{"a": 1, "b": 3}', {"Content-Type": "application/json"} + ), + request.Request( + "POST", "http://host.com/", '{"b": 2, "a": 1}', {"content-type": "application/json"} + ), + ), + ( + request.Request("POST", "http://host.com/", req1_body, {"Content-Type": "text/xml"}), + request.Request("POST", "http://host.com/", req2_body, {"content-type": "text/xml"}), + ), + ], +) +def test_body_match_does_not_match(r1, r2): + with pytest.raises(AssertionError): + matchers.body(r1, r2) + + +def test_query_matcher(): + req1 = request.Request("GET", "http://host.com/?a=b&c=d", "", {}) + req2 = request.Request("GET", "http://host.com/?c=d&a=b", "", {}) + assert matchers.query(req1, req2) is None + + req1 = request.Request("GET", "http://host.com/?a=b&a=b&c=d", "", {}) + req2 = request.Request("GET", "http://host.com/?a=b&c=d&a=b", "", {}) + req3 = request.Request("GET", "http://host.com/?c=d&a=b&a=b", "", {}) + assert matchers.query(req1, req2) is None + assert matchers.query(req1, req3) is None + + +def test_matchers(): + assert_matcher("method") + assert_matcher("scheme") + assert_matcher("host") + assert_matcher("port") + assert_matcher("path") + assert_matcher("query") + + +def test_evaluate_matcher_does_match(): + def bool_matcher(r1, r2): + return True + + def assertion_matcher(r1, r2): + assert 1 == 1 + + r1, r2 = None, None + for matcher in [bool_matcher, assertion_matcher]: + match, assertion_msg = matchers._evaluate_matcher(matcher, r1, r2) + assert match is True + assert assertion_msg is None + + +def test_evaluate_matcher_does_not_match(): + def bool_matcher(r1, r2): + return False + + def assertion_matcher(r1, r2): + # This is like the "assert" statement preventing pytest to recompile it + raise AssertionError() + + r1, r2 = None, None + for matcher in [bool_matcher, assertion_matcher]: + match, assertion_msg = matchers._evaluate_matcher(matcher, r1, r2) + assert match is False + assert not assertion_msg + + +def test_evaluate_matcher_does_not_match_with_assert_message(): + def assertion_matcher(r1, r2): + # This is like the "assert" statement preventing pytest to recompile it + raise AssertionError("Failing matcher") + + r1, r2 = None, None + match, assertion_msg = matchers._evaluate_matcher(assertion_matcher, r1, r2) + assert match is False + assert assertion_msg == "Failing matcher" + + +def test_get_assertion_message(): + assert matchers.get_assertion_message(None) is None + assert matchers.get_assertion_message("") == "" + + +def test_get_assertion_message_with_details(): + assertion_msg = "q1=1 != q2=1" + expected = assertion_msg + assert matchers.get_assertion_message(assertion_msg) == expected + + +@pytest.mark.parametrize( + "r1, r2, expected_successes, expected_failures", + [ + ( + request.Request("GET", "http://host.com/p?a=b", "", {}), + request.Request("GET", "http://host.com/p?a=b", "", {}), + ["method", "path"], + [], + ), + ( + request.Request("GET", "http://host.com/p?a=b", "", {}), + request.Request("POST", "http://host.com/p?a=b", "", {}), + ["path"], + ["method"], + ), + ( + request.Request("GET", "http://host.com/p?a=b", "", {}), + request.Request("POST", "http://host.com/path?a=b", "", {}), + [], + ["method", "path"], + ), + ], +) +def test_get_matchers_results(r1, r2, expected_successes, expected_failures): + successes, failures = matchers.get_matchers_results(r1, r2, [matchers.method, matchers.path]) + assert successes == expected_successes + assert len(failures) == len(expected_failures) + for i, expected_failure in enumerate(expected_failures): + assert failures[i][0] == expected_failure + assert failures[i][1] is not None + + +@mock.patch("vcr.matchers.get_matchers_results") +@pytest.mark.parametrize( + "successes, failures, expected_match", + [(["method", "path"], [], True), (["method"], ["path"], False), ([], ["method", "path"], False)], +) +def test_requests_match(mock_get_matchers_results, successes, failures, expected_match): + mock_get_matchers_results.return_value = (successes, failures) + r1 = request.Request("GET", "http://host.com/p?a=b", "", {}) + r2 = request.Request("GET", "http://host.com/p?a=b", "", {}) + match = matchers.requests_match(r1, r2, [matchers.method, matchers.path]) + assert match is expected_match diff --git a/tools/vcrpy/tests/unit/test_migration.py b/tools/vcrpy/tests/unit/test_migration.py new file mode 100644 index 000000000000..63e8aa4f7667 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_migration.py @@ -0,0 +1,47 @@ +import filecmp +import json +import shutil +import yaml + +import vcr.migration + +# Use the libYAML versions if possible +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + + +def test_try_migrate_with_json(tmpdir): + cassette = tmpdir.join("cassette.json").strpath + shutil.copy("tests/fixtures/migration/old_cassette.json", cassette) + assert vcr.migration.try_migrate(cassette) + with open("tests/fixtures/migration/new_cassette.json", "r") as f: + expected_json = json.load(f) + with open(cassette, "r") as f: + actual_json = json.load(f) + assert actual_json == expected_json + + +def test_try_migrate_with_yaml(tmpdir): + cassette = tmpdir.join("cassette.yaml").strpath + shutil.copy("tests/fixtures/migration/old_cassette.yaml", cassette) + assert vcr.migration.try_migrate(cassette) + with open("tests/fixtures/migration/new_cassette.yaml", "r") as f: + expected_yaml = yaml.load(f, Loader=Loader) + with open(cassette, "r") as f: + actual_yaml = yaml.load(f, Loader=Loader) + assert actual_yaml == expected_yaml + + +def test_try_migrate_with_invalid_or_new_cassettes(tmpdir): + cassette = tmpdir.join("cassette").strpath + files = [ + "tests/fixtures/migration/not_cassette.txt", + "tests/fixtures/migration/new_cassette.yaml", + "tests/fixtures/migration/new_cassette.json", + ] + for file_path in files: + shutil.copy(file_path, cassette) + assert not vcr.migration.try_migrate(cassette) + assert filecmp.cmp(cassette, file_path) # shold not change file diff --git a/tools/vcrpy/tests/unit/test_persist.py b/tools/vcrpy/tests/unit/test_persist.py new file mode 100644 index 000000000000..025ad9682d85 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_persist.py @@ -0,0 +1,30 @@ +import pytest + +from vcr.persisters.filesystem import FilesystemPersister +from vcr.serializers import jsonserializer, yamlserializer + + +@pytest.mark.parametrize( + "cassette_path, serializer", + [ + ("tests/fixtures/migration/old_cassette.json", jsonserializer), + ("tests/fixtures/migration/old_cassette.yaml", yamlserializer), + ], +) +def test_load_cassette_with_old_cassettes(cassette_path, serializer): + with pytest.raises(ValueError) as excinfo: + FilesystemPersister.load_cassette(cassette_path, serializer) + assert "run the migration script" in excinfo.exconly() + + +@pytest.mark.parametrize( + "cassette_path, serializer", + [ + ("tests/fixtures/migration/not_cassette.txt", jsonserializer), + ("tests/fixtures/migration/not_cassette.txt", yamlserializer), + ], +) +def test_load_cassette_with_invalid_cassettes(cassette_path, serializer): + with pytest.raises(Exception) as excinfo: + FilesystemPersister.load_cassette(cassette_path, serializer) + assert "run the migration script" not in excinfo.exconly() diff --git a/tools/vcrpy/tests/unit/test_request.py b/tools/vcrpy/tests/unit/test_request.py new file mode 100644 index 000000000000..19e0841684d2 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_request.py @@ -0,0 +1,86 @@ +import pytest + +from vcr.request import Request, HeadersDict + + +@pytest.mark.parametrize( + "method, uri, expected_str", + [ + ("GET", "http://www.google.com/", ""), + ("OPTIONS", "*", ""), + ("CONNECT", "host.some.where:1234", ""), + ], +) +def test_str(method, uri, expected_str): + assert str(Request(method, uri, "", {})) == expected_str + + +def test_headers(): + headers = {"X-Header1": ["h1"], "X-Header2": "h2"} + req = Request("GET", "http://go.com/", "", headers) + assert req.headers == {"X-Header1": "h1", "X-Header2": "h2"} + req.headers["X-Header1"] = "h11" + assert req.headers == {"X-Header1": "h11", "X-Header2": "h2"} + + +def test_add_header_deprecated(): + req = Request("GET", "http://go.com/", "", {}) + pytest.deprecated_call(req.add_header, "foo", "bar") + assert req.headers == {"foo": "bar"} + + +@pytest.mark.parametrize( + "uri, expected_port", + [ + ("http://go.com/", 80), + ("http://go.com:80/", 80), + ("http://go.com:3000/", 3000), + ("https://go.com/", 443), + ("https://go.com:443/", 443), + ("https://go.com:3000/", 3000), + ("*", None), + ], +) +def test_port(uri, expected_port): + req = Request("GET", uri, "", {}) + assert req.port == expected_port + + +@pytest.mark.parametrize( + "method, uri", + [ + ("GET", "http://go.com/"), + ("GET", "http://go.com:80/"), + ("CONNECT", "localhost:1234"), + ("OPTIONS", "*"), + ], +) +def test_uri(method, uri): + assert Request(method, uri, "", {}).uri == uri + + +def test_HeadersDict(): + + # Simple test of CaseInsensitiveDict + h = HeadersDict() + assert h == {} + h["Content-Type"] = "application/json" + assert h == {"Content-Type": "application/json"} + assert h["content-type"] == "application/json" + assert h["CONTENT-TYPE"] == "application/json" + + # Test feature of HeadersDict: devolve list to first element + h = HeadersDict() + assert h == {} + h["x"] = ["foo", "bar"] + assert h == {"x": "foo"} + + # Test feature of HeadersDict: preserve original key case + h = HeadersDict() + assert h == {} + h["Content-Type"] = "application/json" + assert h == {"Content-Type": "application/json"} + h["content-type"] = "text/plain" + assert h == {"Content-Type": "text/plain"} + h["CONtent-tyPE"] = "whoa" + assert h == {"Content-Type": "whoa"} diff --git a/tools/vcrpy/tests/unit/test_response.py b/tools/vcrpy/tests/unit/test_response.py new file mode 100644 index 000000000000..8a7fdc49a806 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_response.py @@ -0,0 +1,103 @@ +# coding: UTF-8 +import io +import unittest + +import six + +from vcr.stubs import VCRHTTPResponse + + +def test_response_should_have_headers_field(): + recorded_response = { + "status": {"message": "OK", "code": 200}, + "headers": { + "content-length": ["0"], + "server": ["gunicorn/18.0"], + "connection": ["Close"], + "access-control-allow-credentials": ["true"], + "date": ["Fri, 24 Oct 2014 18:35:37 GMT"], + "access-control-allow-origin": ["*"], + "content-type": ["text/html; charset=utf-8"], + }, + "body": {"string": b""}, + } + response = VCRHTTPResponse(recorded_response) + + assert response.headers is not None + + +def test_response_headers_should_be_equal_to_msg(): + recorded_response = { + "status": {"message": b"OK", "code": 200}, + "headers": { + "content-length": ["0"], + "server": ["gunicorn/18.0"], + "connection": ["Close"], + "content-type": ["text/html; charset=utf-8"], + }, + "body": {"string": b""}, + } + response = VCRHTTPResponse(recorded_response) + + assert response.headers == response.msg + + +def test_response_headers_should_have_correct_values(): + recorded_response = { + "status": {"message": "OK", "code": 200}, + "headers": { + "content-length": ["10806"], + "date": ["Fri, 24 Oct 2014 18:35:37 GMT"], + "content-type": ["text/html; charset=utf-8"], + }, + "body": {"string": b""}, + } + response = VCRHTTPResponse(recorded_response) + + assert response.headers.get("content-length") == "10806" + assert response.headers.get("date") == "Fri, 24 Oct 2014 18:35:37 GMT" + + +@unittest.skipIf(six.PY2, "Regression test for Python3 only") +def test_response_parses_correctly_and_fp_attribute_error_is_not_thrown(): + """ + Regression test for https://github.com/kevin1024/vcrpy/issues/440 + :return: + """ + recorded_response = { + "status": {"message": "OK", "code": 200}, + "headers": { + "content-length": ["0"], + "server": ["gunicorn/18.0"], + "connection": ["Close"], + "access-control-allow-credentials": ["true"], + "date": ["Fri, 24 Oct 2014 18:35:37 GMT"], + "access-control-allow-origin": ["*"], + "content-type": ["text/html; charset=utf-8"], + }, + "body": { + "string": b"\nPMID- 19416910\nOWN - NLM\nSTAT- MEDLINE\nDA - 20090513\nDCOM- " + b"20090622\nLR - " + b"20141209\nIS - 1091-6490 (Electronic)\nIS - 0027-8424 (Linking)\nVI - " + b"106\nIP - " + b"19\nDP - 2009 May 12\nTI - Genetic dissection of histone deacetylase " + b"requirement in " + b"tumor cells.\nPG - 7751-5\nLID - 10.1073/pnas.0903139106 [doi]\nAB - " + b"Histone " + b"deacetylase inhibitors (HDACi) represent a new group of drugs currently\n " + b" being " + b"tested in a wide variety of clinical applications. They are especially\n " + b" effective " + b"in preclinical models of cancer where they show antiproliferative\n " + b"action in many " + b"different types of cancer cells. Recently, the first HDACi was\n " + b"approved for the " + b"treatment of cutaneous T cell lymphomas. Most HDACi currently in\n " + b"clinical " + }, + } + vcr_response = VCRHTTPResponse(recorded_response) + handle = io.TextIOWrapper(io.BufferedReader(vcr_response), encoding="utf-8") + handle = iter(handle) + articles = [line for line in handle] + assert len(articles) > 1 diff --git a/tools/vcrpy/tests/unit/test_serialize.py b/tools/vcrpy/tests/unit/test_serialize.py new file mode 100644 index 000000000000..a0d9437859c6 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_serialize.py @@ -0,0 +1,119 @@ +# -*- encoding: utf-8 -*- +import pytest + +from vcr.compat import mock +from vcr.request import Request +from vcr.serialize import deserialize, serialize +from vcr.serializers import yamlserializer, jsonserializer, compat + + +def test_deserialize_old_yaml_cassette(): + with open("tests/fixtures/migration/old_cassette.yaml", "r") as f: + with pytest.raises(ValueError): + deserialize(f.read(), yamlserializer) + + +def test_deserialize_old_json_cassette(): + with open("tests/fixtures/migration/old_cassette.json", "r") as f: + with pytest.raises(ValueError): + deserialize(f.read(), jsonserializer) + + +def test_deserialize_new_yaml_cassette(): + with open("tests/fixtures/migration/new_cassette.yaml", "r") as f: + deserialize(f.read(), yamlserializer) + + +def test_deserialize_new_json_cassette(): + with open("tests/fixtures/migration/new_cassette.json", "r") as f: + deserialize(f.read(), jsonserializer) + + +REQBODY_TEMPLATE = u"""\ +interactions: +- request: + body: {req_body} + headers: + Content-Type: [application/x-www-form-urlencoded] + Host: [httpbin.org] + method: POST + uri: http://httpbin.org/post + response: + body: {{string: ""}} + headers: + content-length: ['0'] + content-type: [application/json] + status: {{code: 200, message: OK}} +""" + + +# A cassette generated under Python 2 stores the request body as a string, +# but the same cassette generated under Python 3 stores it as "!!binary". +# Make sure we accept both forms, regardless of whether we're running under +# Python 2 or 3. +@pytest.mark.parametrize( + "req_body, expect", + [ + # Cassette written under Python 2 (pure ASCII body) + ("x=5&y=2", b"x=5&y=2"), + # Cassette written under Python 3 (pure ASCII body) + ("!!binary |\n eD01Jnk9Mg==", b"x=5&y=2"), + # Request body has non-ASCII chars (x=föo&y=2), encoded in UTF-8. + ('!!python/str "x=f\\xF6o&y=2"', b"x=f\xc3\xb6o&y=2"), + ("!!binary |\n eD1mw7ZvJnk9Mg==", b"x=f\xc3\xb6o&y=2"), + # Same request body, this time encoded in UTF-16. In this case, we + # write the same YAML file under both Python 2 and 3, so there's only + # one test case here. + ( + "!!binary |\n //54AD0AZgD2AG8AJgB5AD0AMgA=", + b"\xff\xfex\x00=\x00f\x00\xf6\x00o\x00&\x00y\x00=\x002\x00", + ), + # Same again, this time encoded in ISO-8859-1. + ("!!binary |\n eD1m9m8meT0y", b"x=f\xf6o&y=2"), + ], +) +def test_deserialize_py2py3_yaml_cassette(tmpdir, req_body, expect): + cfile = tmpdir.join("test_cassette.yaml") + cfile.write(REQBODY_TEMPLATE.format(req_body=req_body)) + with open(str(cfile)) as f: + (requests, responses) = deserialize(f.read(), yamlserializer) + assert requests[0].body == expect + + +@mock.patch.object( + jsonserializer.json, + "dumps", + side_effect=UnicodeDecodeError("utf-8", b"unicode error in serialization", 0, 10, "blew up"), +) +def test_serialize_constructs_UnicodeDecodeError(mock_dumps): + with pytest.raises(UnicodeDecodeError): + jsonserializer.serialize({}) + + +def test_serialize_empty_request(): + request = Request(method="POST", uri="http://localhost/", body="", headers={}) + + serialize({"requests": [request], "responses": [{}]}, jsonserializer) + + +def test_serialize_json_request(): + request = Request(method="POST", uri="http://localhost/", body="{'hello': 'world'}", headers={}) + + serialize({"requests": [request], "responses": [{}]}, jsonserializer) + + +def test_serialize_binary_request(): + msg = "Does this HTTP interaction contain binary data?" + + request = Request(method="POST", uri="http://localhost/", body=b"\x8c", headers={}) + + try: + serialize({"requests": [request], "responses": [{}]}, jsonserializer) + except (UnicodeDecodeError, TypeError) as exc: + assert msg in str(exc) + + +def test_deserialize_no_body_string(): + data = {"body": {"string": None}} + output = compat.convert_to_bytes(data) + assert data == output diff --git a/tools/vcrpy/tests/unit/test_stubs.py b/tools/vcrpy/tests/unit/test_stubs.py new file mode 100644 index 000000000000..e06e4cd92e55 --- /dev/null +++ b/tools/vcrpy/tests/unit/test_stubs.py @@ -0,0 +1,17 @@ +from vcr.stubs import VCRHTTPSConnection +from vcr.compat import mock +from vcr.cassette import Cassette + + +class TestVCRConnection(object): + def test_setting_of_attributes_get_propogated_to_real_connection(self): + vcr_connection = VCRHTTPSConnection("www.examplehost.com") + vcr_connection.ssl_version = "example_ssl_version" + assert vcr_connection.real_connection.ssl_version == "example_ssl_version" + + @mock.patch("vcr.cassette.Cassette.can_play_response_for", return_value=False) + def testing_connect(*args): + vcr_connection = VCRHTTPSConnection("www.google.com") + vcr_connection.cassette = Cassette("test", record_mode="all") + vcr_connection.real_connection.connect() + assert vcr_connection.real_connection.sock is not None diff --git a/tools/vcrpy/tests/unit/test_vcr.py b/tools/vcrpy/tests/unit/test_vcr.py new file mode 100644 index 000000000000..4bdad8d4e8ac --- /dev/null +++ b/tools/vcrpy/tests/unit/test_vcr.py @@ -0,0 +1,362 @@ +import os + +import pytest +from six.moves import http_client as httplib + +from vcr import VCR, use_cassette +from vcr.compat import mock +from vcr.request import Request +from vcr.stubs import VCRHTTPSConnection +from vcr.patch import _HTTPConnection, force_reset + + +def test_vcr_use_cassette(): + record_mode = mock.Mock() + test_vcr = VCR(record_mode=record_mode) + with mock.patch( + "vcr.cassette.Cassette.load", return_value=mock.MagicMock(inject=False) + ) as mock_cassette_load: + + @test_vcr.use_cassette("test") + def function(): + pass + + assert mock_cassette_load.call_count == 0 + function() + assert mock_cassette_load.call_args[1]["record_mode"] is record_mode + + # Make sure that calls to function now use cassettes with the + # new filter_header_settings + test_vcr.record_mode = mock.Mock() + function() + assert mock_cassette_load.call_args[1]["record_mode"] == test_vcr.record_mode + + # Ensure that explicitly provided arguments still supercede + # those on the vcr. + new_record_mode = mock.Mock() + + with test_vcr.use_cassette("test", record_mode=new_record_mode) as cassette: + assert cassette.record_mode == new_record_mode + + +def test_vcr_before_record_request_params(): + base_path = "http://httpbin.org/" + + def before_record_cb(request): + if request.path != "/get": + return request + + test_vcr = VCR( + filter_headers=("cookie", ("bert", "ernie")), + before_record_request=before_record_cb, + ignore_hosts=("www.test.com",), + ignore_localhost=True, + filter_query_parameters=("foo", ("tom", "jerry")), + filter_post_data_parameters=("posted", ("no", "trespassing")), + ) + + with test_vcr.use_cassette("test") as cassette: + # Test explicit before_record_cb + request_get = Request("GET", base_path + "get", "", {}) + assert cassette.filter_request(request_get) is None + request = Request("GET", base_path + "get2", "", {}) + assert cassette.filter_request(request) is not None + + # Test filter_query_parameters + request = Request("GET", base_path + "?foo=bar", "", {}) + assert cassette.filter_request(request).query == [] + request = Request("GET", base_path + "?tom=nobody", "", {}) + assert cassette.filter_request(request).query == [("tom", "jerry")] + + # Test filter_headers + request = Request( + "GET", base_path + "?foo=bar", "", {"cookie": "test", "other": "fun", "bert": "nobody"} + ) + assert cassette.filter_request(request).headers == {"other": "fun", "bert": "ernie"} + + # Test ignore_hosts + request = Request("GET", "http://www.test.com" + "?foo=bar", "", {"cookie": "test", "other": "fun"}) + assert cassette.filter_request(request) is None + + # Test ignore_localhost + request = Request("GET", "http://localhost:8000" + "?foo=bar", "", {"cookie": "test", "other": "fun"}) + assert cassette.filter_request(request) is None + + with test_vcr.use_cassette("test", before_record_request=None) as cassette: + # Test that before_record can be overwritten in context manager. + assert cassette.filter_request(request_get) is not None + + +def test_vcr_before_record_response_iterable(): + # Regression test for #191 + + request = Request("GET", "/", "", {}) + response = object() # just can't be None + + # Prevent actually saving the cassette + with mock.patch("vcr.cassette.FilesystemPersister.save_cassette"): + + # Baseline: non-iterable before_record_response should work + mock_filter = mock.Mock() + vcr = VCR(before_record_response=mock_filter) + with vcr.use_cassette("test") as cassette: + assert mock_filter.call_count == 0 + cassette.append(request, response) + assert mock_filter.call_count == 1 + + # Regression test: iterable before_record_response should work too + mock_filter = mock.Mock() + vcr = VCR(before_record_response=(mock_filter,)) + with vcr.use_cassette("test") as cassette: + assert mock_filter.call_count == 0 + cassette.append(request, response) + assert mock_filter.call_count == 1 + + +def test_before_record_response_as_filter(): + request = Request("GET", "/", "", {}) + response = object() # just can't be None + + # Prevent actually saving the cassette + with mock.patch("vcr.cassette.FilesystemPersister.save_cassette"): + + filter_all = mock.Mock(return_value=None) + vcr = VCR(before_record_response=filter_all) + with vcr.use_cassette("test") as cassette: + cassette.append(request, response) + assert cassette.data == [] + assert not cassette.dirty + + +def test_vcr_path_transformer(): + # Regression test for #199 + + # Prevent actually saving the cassette + with mock.patch("vcr.cassette.FilesystemPersister.save_cassette"): + + # Baseline: path should be unchanged + vcr = VCR() + with vcr.use_cassette("test") as cassette: + assert cassette._path == "test" + + # Regression test: path_transformer=None should do the same. + vcr = VCR(path_transformer=None) + with vcr.use_cassette("test") as cassette: + assert cassette._path == "test" + + # and it should still work with cassette_library_dir + vcr = VCR(cassette_library_dir="/foo") + with vcr.use_cassette("test") as cassette: + assert os.path.abspath(cassette._path) == os.path.abspath("/foo/test") + + +@pytest.fixture +def random_fixture(): + return 1 + + +@use_cassette("test") +def test_fixtures_with_use_cassette(random_fixture): + # Applying a decorator to a test function that requests features can cause + # problems if the decorator does not preserve the signature of the original + # test function. + + # This test ensures that use_cassette preserves the signature of + # the original test function, and thus that use_cassette is + # compatible with py.test fixtures. It is admittedly a bit strange + # because the test would never even run if the relevant feature + # were broken. + pass + + +def test_custom_patchers(): + class Test(object): + attribute = None + attribute2 = None + + test_vcr = VCR(custom_patches=((Test, "attribute", VCRHTTPSConnection),)) + with test_vcr.use_cassette("custom_patches"): + assert issubclass(Test.attribute, VCRHTTPSConnection) + assert VCRHTTPSConnection is not Test.attribute + + with test_vcr.use_cassette("custom_patches", custom_patches=((Test, "attribute2", VCRHTTPSConnection),)): + assert issubclass(Test.attribute, VCRHTTPSConnection) + assert VCRHTTPSConnection is not Test.attribute + assert Test.attribute is Test.attribute2 + + +def test_inject_cassette(): + vcr = VCR(inject_cassette=True) + + @vcr.use_cassette("test", record_mode="once") + def with_cassette_injected(cassette): + assert cassette.record_mode == "once" + + @vcr.use_cassette("test", record_mode="once", inject_cassette=False) + def without_cassette_injected(): + pass + + with_cassette_injected() + without_cassette_injected() + + +def test_with_current_defaults(): + vcr = VCR(inject_cassette=True, record_mode="once") + + @vcr.use_cassette("test", with_current_defaults=False) + def changing_defaults(cassette, checks): + checks(cassette) + + @vcr.use_cassette("test", with_current_defaults=True) + def current_defaults(cassette, checks): + checks(cassette) + + def assert_record_mode_once(cassette): + assert cassette.record_mode == "once" + + def assert_record_mode_all(cassette): + assert cassette.record_mode == "all" + + changing_defaults(assert_record_mode_once) + current_defaults(assert_record_mode_once) + + vcr.record_mode = "all" + changing_defaults(assert_record_mode_all) + current_defaults(assert_record_mode_once) + + +def test_cassette_library_dir_with_decoration_and_no_explicit_path(): + library_dir = "/libary_dir" + vcr = VCR(inject_cassette=True, cassette_library_dir=library_dir) + + @vcr.use_cassette() + def function_name(cassette): + assert cassette._path == os.path.join(library_dir, "function_name") + + function_name() + + +def test_cassette_library_dir_with_decoration_and_explicit_path(): + library_dir = "/libary_dir" + vcr = VCR(inject_cassette=True, cassette_library_dir=library_dir) + + @vcr.use_cassette(path="custom_name") + def function_name(cassette): + assert cassette._path == os.path.join(library_dir, "custom_name") + + function_name() + + +def test_cassette_library_dir_with_decoration_and_super_explicit_path(): + library_dir = "/libary_dir" + vcr = VCR(inject_cassette=True, cassette_library_dir=library_dir) + + @vcr.use_cassette(path=os.path.join(library_dir, "custom_name")) + def function_name(cassette): + assert cassette._path == os.path.join(library_dir, "custom_name") + + function_name() + + +def test_cassette_library_dir_with_path_transformer(): + library_dir = "/libary_dir" + vcr = VCR( + inject_cassette=True, cassette_library_dir=library_dir, path_transformer=lambda path: path + ".json" + ) + + @vcr.use_cassette() + def function_name(cassette): + assert cassette._path == os.path.join(library_dir, "function_name.json") + + function_name() + + +def test_use_cassette_with_no_extra_invocation(): + vcr = VCR(inject_cassette=True, cassette_library_dir="/") + + @vcr.use_cassette + def function_name(cassette): + assert cassette._path == os.path.join("/", "function_name") + + function_name() + + +def test_path_transformer(): + vcr = VCR(inject_cassette=True, cassette_library_dir="/", path_transformer=lambda x: x + "_test") + + @vcr.use_cassette + def function_name(cassette): + assert cassette._path == os.path.join("/", "function_name_test") + + function_name() + + +def test_cassette_name_generator_defaults_to_using_module_function_defined_in(): + vcr = VCR(inject_cassette=True) + + @vcr.use_cassette + def function_name(cassette): + assert cassette._path == os.path.join(os.path.dirname(__file__), "function_name") + + function_name() + + +def test_ensure_suffix(): + vcr = VCR(inject_cassette=True, path_transformer=VCR.ensure_suffix(".yaml")) + + @vcr.use_cassette + def function_name(cassette): + assert cassette._path == os.path.join(os.path.dirname(__file__), "function_name.yaml") + + function_name() + + +def test_additional_matchers(): + vcr = VCR(match_on=("uri",), inject_cassette=True) + + @vcr.use_cassette + def function_defaults(cassette): + assert set(cassette._match_on) == {vcr.matchers["uri"]} + + @vcr.use_cassette(additional_matchers=("body",)) + def function_additional(cassette): + assert set(cassette._match_on) == {vcr.matchers["uri"], vcr.matchers["body"]} + + function_defaults() + function_additional() + + +def test_decoration_should_respect_function_return_value(): + vcr = VCR() + ret = "a-return-value" + + @vcr.use_cassette + def function_with_return(): + return ret + + assert ret == function_with_return() + + +class TestVCRClass(VCR().test_case()): + def no_decoration(self): + assert httplib.HTTPConnection == _HTTPConnection + self.test_dynamically_added() + assert httplib.HTTPConnection == _HTTPConnection + + def test_one(self): + with force_reset(): + self.no_decoration() + with force_reset(): + self.test_two() + assert httplib.HTTPConnection != _HTTPConnection + + def test_two(self): + assert httplib.HTTPConnection != _HTTPConnection + + +def test_dynamically_added(self): + assert httplib.HTTPConnection != _HTTPConnection + + +TestVCRClass.test_dynamically_added = test_dynamically_added +del test_dynamically_added diff --git a/tools/vcrpy/tests/unit/test_vcr_import.py b/tools/vcrpy/tests/unit/test_vcr_import.py new file mode 100644 index 000000000000..5872010aed0b --- /dev/null +++ b/tools/vcrpy/tests/unit/test_vcr_import.py @@ -0,0 +1,16 @@ +import sys + + +def test_vcr_import_deprecation(recwarn): + + if "vcr" in sys.modules: + # Remove imported module entry if already loaded in another test + del sys.modules["vcr"] + + import vcr # noqa: F401 + + if sys.version_info[0] == 2: + assert len(recwarn) == 1 + assert issubclass(recwarn[0].category, DeprecationWarning) + else: + assert len(recwarn) == 0 diff --git a/tools/vcrpy/tox.ini b/tools/vcrpy/tox.ini new file mode 100644 index 000000000000..3f311d1d9a5e --- /dev/null +++ b/tools/vcrpy/tox.ini @@ -0,0 +1,71 @@ +[tox] +skip_missing_interpreters=true +envlist = + cov-clean, + lint, + {py27,py35,py36,py37,py38,pypy,pypy3}-{requests,httplib2,urllib3,tornado4,boto3}, + {py35,py36,py37,py38}-{aiohttp}, + cov-report + + +# Coverage environment tasks: cov-clean and cov-report +# https://pytest-cov.readthedocs.io/en/latest/tox.html +[testenv:cov-clean] +deps = coverage +skip_install=true +commands = coverage erase + +[testenv:cov-report] +deps = coverage +skip_install=true +commands = + coverage html + coverage report --fail-under=90 + +[testenv:lint] +skipsdist = True +commands = + black --version + black --check --diff . + flake8 --version + flake8 --exclude=./docs/conf.py,./.tox/ + pyflakes ./docs/conf.py +deps = + flake8 + black + +[testenv] +# Need to use develop install so that paths +# for aggregate code coverage combine +usedevelop=true +commands = + ./runtests.sh --cov=./vcr --cov-branch --cov-report=xml --cov-append {posargs} +deps = + Flask + mock + pytest + pytest-httpbin + pytest-cov + PyYAML + ipaddress + requests: requests>=2.22.0 + httplib2: httplib2 + urllib3: urllib3 + {py27,py35,py36,pypy}-tornado4: tornado>=4,<5 + {py27,py35,py36,pypy}-tornado4: pytest-tornado + {py27,py35,py36}-tornado4: pycurl + boto3: boto3 + boto3: urllib3 + aiohttp: aiohttp + aiohttp: pytest-asyncio + aiohttp: pytest-aiohttp +depends = + {py27,py35,py36,py37,pypy}-{lint,requests,httplib2,urllib3,tornado4,boto3},{py35,py36,py37}-{aiohttp}: cov-clean + cov-report: {py27,py35,py36,py37,pypy}-{lint,requests,httplib2,urllib3,tornado4,boto3},{py35,py36,py37}-{aiohttp} +passenv = + AWS_ACCESS_KEY_ID + AWS_DEFAULT_REGION + AWS_SECRET_ACCESS_KEY + +[flake8] +max_line_length = 110 diff --git a/tools/vcrpy/vcr/__init__.py b/tools/vcrpy/vcr/__init__.py new file mode 100644 index 000000000000..ed390c5c7a44 --- /dev/null +++ b/tools/vcrpy/vcr/__init__.py @@ -0,0 +1,26 @@ +import logging +import warnings +import sys +from .config import VCR + +# Set default logging handler to avoid "No handler found" warnings. +try: # Python 2.7+ + from logging import NullHandler +except ImportError: + + class NullHandler(logging.Handler): + def emit(self, record): + pass + + +if sys.version_info[0] == 2: + warnings.warn( + "Python 2.x support of vcrpy is deprecated and will be removed in an upcoming major release.", + DeprecationWarning, + ) + +logging.getLogger(__name__).addHandler(NullHandler()) + + +default_vcr = VCR() +use_cassette = default_vcr.use_cassette diff --git a/tools/vcrpy/vcr/_handle_coroutine.py b/tools/vcrpy/vcr/_handle_coroutine.py new file mode 100644 index 000000000000..7dc3befcb9a1 --- /dev/null +++ b/tools/vcrpy/vcr/_handle_coroutine.py @@ -0,0 +1,3 @@ +async def handle_coroutine(vcr, fn): # noqa: E999 + with vcr as cassette: + return await fn(cassette) # noqa: E999 diff --git a/tools/vcrpy/vcr/cassette.py b/tools/vcrpy/vcr/cassette.py new file mode 100644 index 000000000000..bb291091f871 --- /dev/null +++ b/tools/vcrpy/vcr/cassette.py @@ -0,0 +1,360 @@ +import collections +import copy +import sys +import inspect +import logging + +import wrapt + +from .compat import contextlib +from .errors import UnhandledHTTPRequestError +from .matchers import requests_match, uri, method, get_matchers_results +from .patch import CassettePatcherBuilder +from .serializers import yamlserializer +from .persisters.filesystem import FilesystemPersister +from .util import partition_dict + +try: + from asyncio import iscoroutinefunction +except ImportError: + + def iscoroutinefunction(*args, **kwargs): + return False + + +if sys.version_info[:2] >= (3, 5): + from ._handle_coroutine import handle_coroutine +else: + + def handle_coroutine(*args, **kwags): + raise NotImplementedError("Not implemented on Python 2") + + +log = logging.getLogger(__name__) + + +class CassetteContextDecorator(object): + """Context manager/decorator that handles installing the cassette and + removing cassettes. + + This class defers the creation of a new cassette instance until + the point at which it is installed by context manager or + decorator. The fact that a new cassette is used with each + application prevents the state of any cassette from interfering + with another. + + Instances of this class are NOT reentrant as context managers. + However, functions that are decorated by + ``CassetteContextDecorator`` instances ARE reentrant. See the + implementation of ``__call__`` on this class for more details. + There is also a guard against attempts to reenter instances of + this class as a context manager in ``__exit__``. + """ + + _non_cassette_arguments = ("path_transformer", "func_path_generator") + + @classmethod + def from_args(cls, cassette_class, **kwargs): + return cls(cassette_class, lambda: dict(kwargs)) + + def __init__(self, cls, args_getter): + self.cls = cls + self._args_getter = args_getter + self.__finish = None + + def _patch_generator(self, cassette): + with contextlib.ExitStack() as exit_stack: + for patcher in CassettePatcherBuilder(cassette).build(): + exit_stack.enter_context(patcher) + log_format = "{action} context for cassette at {path}." + log.debug(log_format.format(action="Entering", path=cassette._path)) + yield cassette + log.debug(log_format.format(action="Exiting", path=cassette._path)) + # TODO(@IvanMalison): Hmmm. it kind of feels like this should be + # somewhere else. + cassette._save() + + def __enter__(self): + # This assertion is here to prevent the dangerous behavior + # that would result from forgetting about a __finish before + # completing it. + # How might this condition be met? Here is an example: + # context_decorator = Cassette.use('whatever') + # with context_decorator: + # with context_decorator: + # pass + assert self.__finish is None, "Cassette already open." + other_kwargs, cassette_kwargs = partition_dict( + lambda key, _: key in self._non_cassette_arguments, self._args_getter() + ) + if other_kwargs.get("path_transformer"): + transformer = other_kwargs["path_transformer"] + cassette_kwargs["path"] = transformer(cassette_kwargs["path"]) + self.__finish = self._patch_generator(self.cls.load(**cassette_kwargs)) + return next(self.__finish) + + def __exit__(self, *args): + next(self.__finish, None) + self.__finish = None + + @wrapt.decorator + def __call__(self, function, instance, args, kwargs): + # This awkward cloning thing is done to ensure that decorated + # functions are reentrant. This is required for thread + # safety and the correct operation of recursive functions. + args_getter = self._build_args_getter_for_decorator(function) + return type(self)(self.cls, args_getter)._execute_function(function, args, kwargs) + + def _execute_function(self, function, args, kwargs): + def handle_function(cassette): + if cassette.inject: + return function(cassette, *args, **kwargs) + else: + return function(*args, **kwargs) + + if iscoroutinefunction(function): + return handle_coroutine(vcr=self, fn=handle_function) + if inspect.isgeneratorfunction(function): + return self._handle_generator(fn=handle_function) + + return self._handle_function(fn=handle_function) + + def _handle_generator(self, fn): + """Wraps a generator so that we're inside the cassette context for the + duration of the generator. + """ + with self as cassette: + coroutine = fn(cassette) + # We don't need to catch StopIteration. The caller (Tornado's + # gen.coroutine, for example) will handle that. + to_yield = next(coroutine) + while True: + try: + to_send = yield to_yield + except Exception: + to_yield = coroutine.throw(*sys.exc_info()) + else: + try: + to_yield = coroutine.send(to_send) + except StopIteration: + break + + def _handle_function(self, fn): + with self as cassette: + return fn(cassette) + + @staticmethod + def get_function_name(function): + return function.__name__ + + def _build_args_getter_for_decorator(self, function): + def new_args_getter(): + kwargs = self._args_getter() + if "path" not in kwargs: + name_generator = kwargs.get("func_path_generator") or self.get_function_name + path = name_generator(function) + kwargs["path"] = path + return kwargs + + return new_args_getter + + +class Cassette(object): + """A container for recorded requests and responses""" + + @classmethod + def load(cls, **kwargs): + """Instantiate and load the cassette stored at the specified path.""" + new_cassette = cls(**kwargs) + new_cassette._load() + return new_cassette + + @classmethod + def use_arg_getter(cls, arg_getter): + return CassetteContextDecorator(cls, arg_getter) + + @classmethod + def use(cls, **kwargs): + return CassetteContextDecorator.from_args(cls, **kwargs) + + def __init__( + self, + path, + serializer=None, + persister=None, + record_mode="once", + match_on=(uri, method), + before_record_request=None, + before_record_response=None, + custom_patches=(), + inject=False, + ): + self._persister = persister or FilesystemPersister + self._path = path + self._serializer = serializer or yamlserializer + self._match_on = match_on + self._before_record_request = before_record_request or (lambda x: x) + log.info(self._before_record_request) + self._before_record_response = before_record_response or (lambda x: x) + self.inject = inject + self.record_mode = record_mode + self.custom_patches = custom_patches + + # self.data is the list of (req, resp) tuples + self.data = [] + self.play_counts = collections.Counter() + self.dirty = False + self.rewound = False + + @property + def play_count(self): + return sum(self.play_counts.values()) + + @property + def all_played(self): + """Returns True if all responses have been played, False otherwise.""" + return self.play_count == len(self) + + @property + def requests(self): + return [request for (request, response) in self.data] + + @property + def responses(self): + return [response for (request, response) in self.data] + + @property + def write_protected(self): + return self.rewound and self.record_mode == "once" or self.record_mode == "none" + + def append(self, request, response): + """Add a request, response pair to this cassette""" + log.info("Appending request %s and response %s", request, response) + request = self._before_record_request(request) + if not request: + return + # Deepcopy is here because mutation of `response` will corrupt the + # real response. + response = copy.deepcopy(response) + response = self._before_record_response(response) + if response is None: + return + self.data.append((request, response)) + self.dirty = True + + def filter_request(self, request): + return self._before_record_request(request) + + def _responses(self, request): + """ + internal API, returns an iterator with all responses matching + the request. + """ + request = self._before_record_request(request) + for index, (stored_request, response) in enumerate(self.data): + if requests_match(request, stored_request, self._match_on): + yield index, response + + def can_play_response_for(self, request): + request = self._before_record_request(request) + return request and request in self and self.record_mode != "all" and self.rewound + + def play_response(self, request): + """ + Get the response corresponding to a request, but only if it + hasn't been played back before, and mark it as played + """ + for index, response in self._responses(request): + if self.play_counts[index] == 0: + self.play_counts[index] += 1 + return response + # The cassette doesn't contain the request asked for. + raise UnhandledHTTPRequestError( + "The cassette (%r) doesn't contain the request (%r) asked for" % (self._path, request) + ) + + def responses_of(self, request): + """ + Find the responses corresponding to a request. + This function isn't actually used by VCR internally, but is + provided as an external API. + """ + responses = [response for index, response in self._responses(request)] + + if responses: + return responses + # The cassette doesn't contain the request asked for. + raise UnhandledHTTPRequestError( + "The cassette (%r) doesn't contain the request (%r) asked for" % (self._path, request) + ) + + def rewind(self): + self.play_counts = collections.Counter() + + def find_requests_with_most_matches(self, request): + """ + Get the most similar request(s) stored in the cassette + of a given request as a list of tuples like this: + - the request object + - the successful matchers as string + - the failed matchers and the related assertion message with the difference details as strings tuple + + This is useful when a request failed to be found, + we can get the similar request(s) in order to know what have changed in the request parts. + """ + best_matches = [] + request = self._before_record_request(request) + for index, (stored_request, response) in enumerate(self.data): + successes, fails = get_matchers_results(request, stored_request, self._match_on) + best_matches.append((len(successes), stored_request, successes, fails)) + best_matches.sort(key=lambda t: t[0], reverse=True) + # Get the first best matches (multiple if equal matches) + final_best_matches = [] + + if not best_matches: + return final_best_matches + + previous_nb_success = best_matches[0][0] + for best_match in best_matches: + nb_success = best_match[0] + # Do not keep matches that have 0 successes, + # it means that the request is totally different from + # the ones stored in the cassette + if nb_success < 1 or previous_nb_success != nb_success: + break + previous_nb_success = nb_success + final_best_matches.append(best_match[1:]) + + return final_best_matches + + def _as_dict(self): + return {"requests": self.requests, "responses": self.responses} + + def _save(self, force=False): + if force or self.dirty: + self._persister.save_cassette(self._path, self._as_dict(), serializer=self._serializer) + self.dirty = False + + def _load(self): + try: + requests, responses = self._persister.load_cassette(self._path, serializer=self._serializer) + for request, response in zip(requests, responses): + self.append(request, response) + self.dirty = False + self.rewound = True + except ValueError: + pass + + def __str__(self): + return "".format(len(self)) + + def __len__(self): + """Return the number of request,response pairs stored in here""" + return len(self.data) + + def __contains__(self, request): + """Return whether or not a request has been stored""" + for index, response in self._responses(request): + if self.play_counts[index] == 0: + return True + return False diff --git a/tools/vcrpy/vcr/compat.py b/tools/vcrpy/vcr/compat.py new file mode 100644 index 000000000000..1480928b3d98 --- /dev/null +++ b/tools/vcrpy/vcr/compat.py @@ -0,0 +1,14 @@ +try: + from unittest import mock +except ImportError: + import mock + +try: + import contextlib +except ImportError: + import contextlib2 as contextlib +else: + if not hasattr(contextlib, "ExitStack"): + import contextlib2 as contextlib + +__all__ = ["mock", "contextlib"] diff --git a/tools/vcrpy/vcr/config.py b/tools/vcrpy/vcr/config.py new file mode 100644 index 000000000000..5c63837c41ba --- /dev/null +++ b/tools/vcrpy/vcr/config.py @@ -0,0 +1,254 @@ +import copy + +try: + from collections import abc as collections_abc # only works on python 3.3+ +except ImportError: + import collections as collections_abc +import functools +import inspect +import os +import types + +import six + +from .cassette import Cassette +from .serializers import yamlserializer, jsonserializer +from .persisters.filesystem import FilesystemPersister +from .util import compose, auto_decorate +from . import matchers +from . import filters + + +class VCR(object): + @staticmethod + def is_test_method(method_name, function): + return method_name.startswith("test") and isinstance(function, types.FunctionType) + + @staticmethod + def ensure_suffix(suffix): + def ensure(path): + if not path.endswith(suffix): + return path + suffix + return path + + return ensure + + def __init__( + self, + path_transformer=None, + before_record_request=None, + custom_patches=(), + filter_query_parameters=(), + ignore_hosts=(), + record_mode="once", + ignore_localhost=False, + filter_headers=(), + before_record_response=None, + filter_post_data_parameters=(), + match_on=("method", "scheme", "host", "port", "path", "query"), + before_record=None, + inject_cassette=False, + serializer="yaml", + cassette_library_dir=None, + func_path_generator=None, + decode_compressed_response=False, + ): + self.serializer = serializer + self.match_on = match_on + self.cassette_library_dir = cassette_library_dir + self.serializers = {"yaml": yamlserializer, "json": jsonserializer} + self.matchers = { + "method": matchers.method, + "uri": matchers.uri, + "url": matchers.uri, # matcher for backwards compatibility + "scheme": matchers.scheme, + "host": matchers.host, + "port": matchers.port, + "path": matchers.path, + "query": matchers.query, + "headers": matchers.headers, + "raw_body": matchers.raw_body, + "body": matchers.body, + } + self.persister = FilesystemPersister + self.record_mode = record_mode + self.filter_headers = filter_headers + self.filter_query_parameters = filter_query_parameters + self.filter_post_data_parameters = filter_post_data_parameters + self.before_record_request = before_record_request or before_record + self.before_record_response = before_record_response + self.ignore_hosts = ignore_hosts + self.ignore_localhost = ignore_localhost + self.inject_cassette = inject_cassette + self.path_transformer = path_transformer + self.func_path_generator = func_path_generator + self.decode_compressed_response = decode_compressed_response + self._custom_patches = tuple(custom_patches) + + def _get_serializer(self, serializer_name): + try: + serializer = self.serializers[serializer_name] + except KeyError: + raise KeyError("Serializer {} doesn't exist or isn't registered".format(serializer_name)) + return serializer + + def _get_matchers(self, matcher_names): + matchers = [] + try: + for m in matcher_names: + matchers.append(self.matchers[m]) + except KeyError: + raise KeyError("Matcher {} doesn't exist or isn't registered".format(m)) + return matchers + + def use_cassette(self, path=None, **kwargs): + if path is not None and not isinstance(path, six.string_types): + function = path + # Assume this is an attempt to decorate a function + return self._use_cassette(**kwargs)(function) + return self._use_cassette(path=path, **kwargs) + + def _use_cassette(self, with_current_defaults=False, **kwargs): + if with_current_defaults: + config = self.get_merged_config(**kwargs) + return Cassette.use(**config) + # This is made a function that evaluates every time a cassette + # is made so that changes that are made to this VCR instance + # that occur AFTER the `use_cassette` decorator is applied + # still affect subsequent calls to the decorated function. + args_getter = functools.partial(self.get_merged_config, **kwargs) + return Cassette.use_arg_getter(args_getter) + + def get_merged_config(self, **kwargs): + serializer_name = kwargs.get("serializer", self.serializer) + matcher_names = kwargs.get("match_on", self.match_on) + path_transformer = kwargs.get("path_transformer", self.path_transformer) + func_path_generator = kwargs.get("func_path_generator", self.func_path_generator) + cassette_library_dir = kwargs.get("cassette_library_dir", self.cassette_library_dir) + additional_matchers = kwargs.get("additional_matchers", ()) + + if cassette_library_dir: + + def add_cassette_library_dir(path): + if not path.startswith(cassette_library_dir): + return os.path.join(cassette_library_dir, path) + return path + + path_transformer = compose(add_cassette_library_dir, path_transformer) + elif not func_path_generator: + # If we don't have a library dir, use the functions + # location to build a full path for cassettes. + func_path_generator = self._build_path_from_func_using_module + + merged_config = { + "serializer": self._get_serializer(serializer_name), + "persister": self.persister, + "match_on": self._get_matchers(tuple(matcher_names) + tuple(additional_matchers)), + "record_mode": kwargs.get("record_mode", self.record_mode), + "before_record_request": self._build_before_record_request(kwargs), + "before_record_response": self._build_before_record_response(kwargs), + "custom_patches": self._custom_patches + kwargs.get("custom_patches", ()), + "inject": kwargs.get("inject_cassette", self.inject_cassette), + "path_transformer": path_transformer, + "func_path_generator": func_path_generator, + } + path = kwargs.get("path") + if path: + merged_config["path"] = path + return merged_config + + def _build_before_record_response(self, options): + before_record_response = options.get("before_record_response", self.before_record_response) + decode_compressed_response = options.get( + "decode_compressed_response", self.decode_compressed_response + ) + filter_functions = [] + if decode_compressed_response: + filter_functions.append(filters.decode_response) + if before_record_response: + if not isinstance(before_record_response, collections_abc.Iterable): + before_record_response = (before_record_response,) + filter_functions.extend(before_record_response) + + def before_record_response(response): + for function in filter_functions: + if response is None: + break + response = function(response) + return response + + return before_record_response + + def _build_before_record_request(self, options): + filter_functions = [] + filter_headers = options.get("filter_headers", self.filter_headers) + filter_query_parameters = options.get("filter_query_parameters", self.filter_query_parameters) + filter_post_data_parameters = options.get( + "filter_post_data_parameters", self.filter_post_data_parameters + ) + before_record_request = options.get( + "before_record_request", options.get("before_record", self.before_record_request) + ) + ignore_hosts = options.get("ignore_hosts", self.ignore_hosts) + ignore_localhost = options.get("ignore_localhost", self.ignore_localhost) + if filter_headers: + replacements = [h if isinstance(h, tuple) else (h, None) for h in filter_headers] + filter_functions.append(functools.partial(filters.replace_headers, replacements=replacements)) + if filter_query_parameters: + replacements = [p if isinstance(p, tuple) else (p, None) for p in filter_query_parameters] + filter_functions.append( + functools.partial(filters.replace_query_parameters, replacements=replacements) + ) + if filter_post_data_parameters: + replacements = [p if isinstance(p, tuple) else (p, None) for p in filter_post_data_parameters] + filter_functions.append( + functools.partial(filters.replace_post_data_parameters, replacements=replacements) + ) + + hosts_to_ignore = set(ignore_hosts) + if ignore_localhost: + hosts_to_ignore.update(("localhost", "0.0.0.0", "127.0.0.1")) + if hosts_to_ignore: + filter_functions.append(self._build_ignore_hosts(hosts_to_ignore)) + + if before_record_request: + if not isinstance(before_record_request, collections_abc.Iterable): + before_record_request = (before_record_request,) + filter_functions.extend(before_record_request) + + def before_record_request(request): + request = copy.copy(request) + for function in filter_functions: + if request is None: + break + request = function(request) + return request + + return before_record_request + + @staticmethod + def _build_ignore_hosts(hosts_to_ignore): + def filter_ignored_hosts(request): + if hasattr(request, "host") and request.host in hosts_to_ignore: + return + return request + + return filter_ignored_hosts + + @staticmethod + def _build_path_from_func_using_module(function): + return os.path.join(os.path.dirname(inspect.getfile(function)), function.__name__) + + def register_serializer(self, name, serializer): + self.serializers[name] = serializer + + def register_matcher(self, name, matcher): + self.matchers[name] = matcher + + def register_persister(self, persister): + # Singleton, no name required + self.persister = persister + + def test_case(self, predicate=None): + predicate = predicate or self.is_test_method + return six.with_metaclass(auto_decorate(self.use_cassette, predicate)) diff --git a/tools/vcrpy/vcr/errors.py b/tools/vcrpy/vcr/errors.py new file mode 100644 index 000000000000..92926cc4691b --- /dev/null +++ b/tools/vcrpy/vcr/errors.py @@ -0,0 +1,42 @@ +class CannotOverwriteExistingCassetteException(Exception): + def __init__(self, *args, **kwargs): + self.cassette = kwargs["cassette"] + self.failed_request = kwargs["failed_request"] + message = self._get_message(kwargs["cassette"], kwargs["failed_request"]) + super(CannotOverwriteExistingCassetteException, self).__init__(message) + + @staticmethod + def _get_message(cassette, failed_request): + """Get the final message related to the exception""" + # Get the similar requests in the cassette that + # have match the most with the request. + best_matches = cassette.find_requests_with_most_matches(failed_request) + if best_matches: + # Build a comprehensible message to put in the exception. + best_matches_msg = "Found {} similar requests with {} different matcher(s) :\n".format( + len(best_matches), len(best_matches[0][2]) + ) + + for idx, best_match in enumerate(best_matches, start=1): + request, succeeded_matchers, failed_matchers_assertion_msgs = best_match + best_matches_msg += ( + "\n%s - (%r).\n" + "Matchers succeeded : %s\n" + "Matchers failed :\n" % (idx, request, succeeded_matchers) + ) + for failed_matcher, assertion_msg in failed_matchers_assertion_msgs: + best_matches_msg += "%s - assertion failure :\n" "%s\n" % (failed_matcher, assertion_msg) + else: + best_matches_msg = "No similar requests, that have not been played, found." + return ( + "Can't overwrite existing cassette (%r) in " + "your current record mode (%r).\n" + "No match for the request (%r) was found.\n" + "%s" % (cassette._path, cassette.record_mode, failed_request, best_matches_msg) + ) + + +class UnhandledHTTPRequestError(KeyError): + """Raised when a cassette does not contain the request we want.""" + + pass diff --git a/tools/vcrpy/vcr/filters.py b/tools/vcrpy/vcr/filters.py new file mode 100644 index 000000000000..83af946fc56f --- /dev/null +++ b/tools/vcrpy/vcr/filters.py @@ -0,0 +1,166 @@ +from six import BytesIO, text_type +from six.moves.urllib.parse import urlparse, urlencode, urlunparse +import copy +import json +import zlib + +from .util import CaseInsensitiveDict + + +def replace_headers(request, replacements): + """ + Replace headers in request according to replacements. The replacements + should be a list of (key, value) pairs where the value can be any of: + 1. A simple replacement string value. + 2. None to remove the given header. + 3. A callable which accepts (key, value, request) and returns a string + value or None. + """ + new_headers = request.headers.copy() + for k, rv in replacements: + if k in new_headers: + ov = new_headers.pop(k) + if callable(rv): + rv = rv(key=k, value=ov, request=request) + if rv is not None: + new_headers[k] = rv + request.headers = new_headers + return request + + +def remove_headers(request, headers_to_remove): + """ + Wrap replace_headers() for API backward compatibility. + """ + replacements = [(k, None) for k in headers_to_remove] + return replace_headers(request, replacements) + + +def replace_query_parameters(request, replacements): + """ + Replace query parameters in request according to replacements. The + replacements should be a list of (key, value) pairs where the value can be + any of: + 1. A simple replacement string value. + 2. None to remove the given header. + 3. A callable which accepts (key, value, request) and returns a string + value or None. + """ + query = request.query + new_query = [] + replacements = dict(replacements) + for k, ov in query: + if k not in replacements: + new_query.append((k, ov)) + else: + rv = replacements[k] + if callable(rv): + rv = rv(key=k, value=ov, request=request) + if rv is not None: + new_query.append((k, rv)) + uri_parts = list(urlparse(request.uri)) + uri_parts[4] = urlencode(new_query) + request.uri = urlunparse(uri_parts) + return request + + +def remove_query_parameters(request, query_parameters_to_remove): + """ + Wrap replace_query_parameters() for API backward compatibility. + """ + replacements = [(k, None) for k in query_parameters_to_remove] + return replace_query_parameters(request, replacements) + + +def replace_post_data_parameters(request, replacements): + """ + Replace post data in request--either form data or json--according to + replacements. The replacements should be a list of (key, value) pairs where + the value can be any of: + 1. A simple replacement string value. + 2. None to remove the given header. + 3. A callable which accepts (key, value, request) and returns a string + value or None. + """ + if not request.body: + # Nothing to replace + return request + + replacements = dict(replacements) + if request.method == "POST" and not isinstance(request.body, BytesIO): + if request.headers.get("Content-Type") == "application/json": + json_data = json.loads(request.body.decode("utf-8")) + for k, rv in replacements.items(): + if k in json_data: + ov = json_data.pop(k) + if callable(rv): + rv = rv(key=k, value=ov, request=request) + if rv is not None: + json_data[k] = rv + request.body = json.dumps(json_data).encode("utf-8") + else: + if isinstance(request.body, text_type): + request.body = request.body.encode("utf-8") + splits = [p.partition(b"=") for p in request.body.split(b"&")] + new_splits = [] + for k, sep, ov in splits: + if sep is None: + new_splits.append((k, sep, ov)) + else: + rk = k.decode("utf-8") + if rk not in replacements: + new_splits.append((k, sep, ov)) + else: + rv = replacements[rk] + if callable(rv): + rv = rv(key=rk, value=ov.decode("utf-8"), request=request) + if rv is not None: + new_splits.append((k, sep, rv.encode("utf-8"))) + request.body = b"&".join(k if sep is None else b"".join([k, sep, v]) for k, sep, v in new_splits) + return request + + +def remove_post_data_parameters(request, post_data_parameters_to_remove): + """ + Wrap replace_post_data_parameters() for API backward compatibility. + """ + replacements = [(k, None) for k in post_data_parameters_to_remove] + return replace_post_data_parameters(request, replacements) + + +def decode_response(response): + """ + If the response is compressed with gzip or deflate: + 1. decompress the response body + 2. delete the content-encoding header + 3. update content-length header to decompressed length + """ + + def is_compressed(headers): + encoding = headers.get("content-encoding", []) + return encoding and encoding[0] in ("gzip", "deflate") + + def decompress_body(body, encoding): + """Returns decompressed body according to encoding using zlib. + to (de-)compress gzip format, use wbits = zlib.MAX_WBITS | 16 + """ + if encoding == "gzip": + return zlib.decompress(body, zlib.MAX_WBITS | 16) + else: # encoding == 'deflate' + return zlib.decompress(body) + + # Deepcopy here in case `headers` contain objects that could + # be mutated by a shallow copy and corrupt the real response. + response = copy.deepcopy(response) + headers = CaseInsensitiveDict(response["headers"]) + if is_compressed(headers): + encoding = headers["content-encoding"][0] + headers["content-encoding"].remove(encoding) + if not headers["content-encoding"]: + del headers["content-encoding"] + + new_body = decompress_body(response["body"]["string"], encoding) + response["body"]["string"] = new_body + headers["content-length"] = [str(len(new_body))] + response["headers"] = dict(headers) + return response diff --git a/tools/vcrpy/vcr/matchers.py b/tools/vcrpy/vcr/matchers.py new file mode 100644 index 000000000000..eabd61f27d64 --- /dev/null +++ b/tools/vcrpy/vcr/matchers.py @@ -0,0 +1,142 @@ +import json +from six.moves import urllib, xmlrpc_client +from .util import read_body +import logging + + +log = logging.getLogger(__name__) + + +def method(r1, r2): + assert r1.method == r2.method, "{} != {}".format(r1.method, r2.method) + + +def uri(r1, r2): + assert r1.uri == r2.uri, "{} != {}".format(r1.uri, r2.uri) + + +def host(r1, r2): + assert r1.host == r2.host, "{} != {}".format(r1.host, r2.host) + + +def scheme(r1, r2): + assert r1.scheme == r2.scheme, "{} != {}".format(r1.scheme, r2.scheme) + + +def port(r1, r2): + assert r1.port == r2.port, "{} != {}".format(r1.port, r2.port) + + +def path(r1, r2): + assert r1.path == r2.path, "{} != {}".format(r1.path, r2.path) + + +def query(r1, r2): + assert r1.query == r2.query, "{} != {}".format(r1.query, r2.query) + + +def raw_body(r1, r2): + assert read_body(r1) == read_body(r2) + + +def body(r1, r2): + transformer = _get_transformer(r1) + r2_transformer = _get_transformer(r2) + if transformer != r2_transformer: + transformer = _identity + assert transformer(read_body(r1)) == transformer(read_body(r2)) + + +def headers(r1, r2): + assert r1.headers == r2.headers, "{} != {}".format(r1.headers, r2.headers) + + +def _header_checker(value, header="Content-Type"): + def checker(headers): + _header = headers.get(header, "") + if isinstance(_header, bytes): + _header = _header.decode("utf-8") + return value in _header.lower() + + return checker + + +def _transform_json(body): + # Request body is always a byte string, but json.loads() wants a text + # string. RFC 7159 says the default encoding is UTF-8 (although UTF-16 + # and UTF-32 are also allowed: hmmmmm). + if body: + return json.loads(body.decode("utf-8")) + + +_xml_header_checker = _header_checker("text/xml") +_xmlrpc_header_checker = _header_checker("xmlrpc", header="User-Agent") +_checker_transformer_pairs = ( + ( + _header_checker("application/x-www-form-urlencoded"), + lambda body: urllib.parse.parse_qs(body.decode("ascii")), + ), + (_header_checker("application/json"), _transform_json), + (lambda request: _xml_header_checker(request) and _xmlrpc_header_checker(request), xmlrpc_client.loads), +) + + +def _identity(x): + return x + + +def _get_transformer(request): + for checker, transformer in _checker_transformer_pairs: + if checker(request.headers): + return transformer + else: + return _identity + + +def requests_match(r1, r2, matchers): + successes, failures = get_matchers_results(r1, r2, matchers) + if failures: + log.debug("Requests {} and {} differ.\n" "Failure details:\n" "{}".format(r1, r2, failures)) + return len(failures) == 0 + + +def _evaluate_matcher(matcher_function, *args): + """ + Evaluate the result of a given matcher as a boolean with an assertion error message if any. + It handles two types of matcher : + - a matcher returning a boolean value. + - a matcher that only makes an assert, returning None or raises an assertion error. + """ + assertion_message = None + try: + match = matcher_function(*args) + match = True if match is None else match + except AssertionError as e: + match = False + assertion_message = str(e) + return match, assertion_message + + +def get_matchers_results(r1, r2, matchers): + """ + Get the comparison results of two requests as two list. + The first returned list represents the matchers names that passed. + The second list is the failed matchers as a string with failed assertion details if any. + """ + matches_success, matches_fails = [], [] + for m in matchers: + matcher_name = m.__name__ + match, assertion_message = _evaluate_matcher(m, r1, r2) + if match: + matches_success.append(matcher_name) + else: + assertion_message = get_assertion_message(assertion_message) + matches_fails.append((matcher_name, assertion_message)) + return matches_success, matches_fails + + +def get_assertion_message(assertion_details): + """ + Get a detailed message about the failing matcher. + """ + return assertion_details diff --git a/tools/vcrpy/vcr/migration.py b/tools/vcrpy/vcr/migration.py new file mode 100644 index 000000000000..89b71e700599 --- /dev/null +++ b/tools/vcrpy/vcr/migration.py @@ -0,0 +1,157 @@ +""" +Migration script for old 'yaml' and 'json' cassettes + +.. warning:: Backup your cassettes files before migration. + +It merges and deletes the request obsolete keys (protocol, host, port, path) +into new 'uri' key. +Usage:: + + python -m vcr.migration PATH + +The PATH can be path to the directory with cassettes or cassette itself +""" + +import json +import os +import shutil +import sys +import tempfile +import yaml + +from .serializers import yamlserializer, jsonserializer +from .serialize import serialize +from . import request +from .stubs.compat import get_httpmessage + +# Use the libYAML versions if possible +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + + +def preprocess_yaml(cassette): + # this is the hack that makes the whole thing work. The old version used + # to deserialize to Request objects automatically using pyYaml's !!python + # tag system. This made it difficult to deserialize old cassettes on new + # versions. So this just strips the tags before deserializing. + + STRINGS_TO_NUKE = [ + "!!python/object:vcr.request.Request", + "!!python/object/apply:__builtin__.frozenset", + "!!python/object/apply:builtins.frozenset", + ] + for s in STRINGS_TO_NUKE: + cassette = cassette.replace(s, "") + return cassette + + +PARTS = ["protocol", "host", "port", "path"] + + +def build_uri(**parts): + port = parts["port"] + scheme = parts["protocol"] + default_port = {"https": 443, "http": 80}[scheme] + parts["port"] = ":{}".format(port) if port != default_port else "" + return "{protocol}://{host}{port}{path}".format(**parts) + + +def _migrate(data): + interactions = [] + for item in data: + req = item["request"] + res = item["response"] + uri = {k: req.pop(k) for k in PARTS} + req["uri"] = build_uri(**uri) + # convert headers to dict of lists + headers = req["headers"] + for k in headers: + headers[k] = [headers[k]] + response_headers = {} + for k, v in get_httpmessage(b"".join(h.encode("utf-8") for h in res["headers"])).items(): + response_headers.setdefault(k, []) + response_headers[k].append(v) + res["headers"] = response_headers + interactions.append({"request": req, "response": res}) + return { + "requests": [request.Request._from_dict(i["request"]) for i in interactions], + "responses": [i["response"] for i in interactions], + } + + +def migrate_json(in_fp, out_fp): + data = json.load(in_fp) + if _already_migrated(data): + return False + interactions = _migrate(data) + out_fp.write(serialize(interactions, jsonserializer)) + return True + + +def _list_of_tuples_to_dict(fs): + return {k: v for k, v in fs[0]} + + +def _already_migrated(data): + try: + if data.get("version") == 1: + return True + except AttributeError: + return False + + +def migrate_yml(in_fp, out_fp): + data = yaml.load(preprocess_yaml(in_fp.read()), Loader=Loader) + if _already_migrated(data): + return False + for i in range(len(data)): + data[i]["request"]["headers"] = _list_of_tuples_to_dict(data[i]["request"]["headers"]) + interactions = _migrate(data) + out_fp.write(serialize(interactions, yamlserializer)) + return True + + +def migrate(file_path, migration_fn): + # because we assume that original files can be reverted + # we will try to copy the content. (os.rename not needed) + with tempfile.TemporaryFile(mode="w+") as out_fp: + with open(file_path, "r") as in_fp: + if not migration_fn(in_fp, out_fp): + return False + with open(file_path, "w") as in_fp: + out_fp.seek(0) + shutil.copyfileobj(out_fp, in_fp) + return True + + +def try_migrate(path): + if path.endswith(".json"): + return migrate(path, migrate_json) + elif path.endswith(".yaml") or path.endswith(".yml"): + return migrate(path, migrate_yml) + return False + + +def main(): + if len(sys.argv) != 2: + raise SystemExit( + "Please provide path to cassettes directory or file. " "Usage: python -m vcr.migration PATH" + ) + + path = sys.argv[1] + if not os.path.isabs(path): + path = os.path.abspath(path) + files = [path] + if os.path.isdir(path): + files = (os.path.join(root, name) for (root, dirs, files) in os.walk(path) for name in files) + for file_path in files: + migrated = try_migrate(file_path) + status = "OK" if migrated else "FAIL" + sys.stderr.write("[{}] {}\n".format(status, file_path)) + sys.stderr.write("Done.\n") + + +if __name__ == "__main__": + main() diff --git a/tools/vcrpy/vcr/patch.py b/tools/vcrpy/vcr/patch.py new file mode 100644 index 000000000000..ae49f7bfca75 --- /dev/null +++ b/tools/vcrpy/vcr/patch.py @@ -0,0 +1,501 @@ +"""Utilities for patching in cassettes""" +import functools +import itertools + +from .compat import contextlib, mock +from .stubs import VCRHTTPConnection, VCRHTTPSConnection +from six.moves import http_client as httplib + +import logging + +log = logging.getLogger(__name__) +# Save some of the original types for the purposes of unpatching +_HTTPConnection = httplib.HTTPConnection +_HTTPSConnection = httplib.HTTPSConnection + +# Try to save the original types for boto3 +try: + from botocore.awsrequest import AWSHTTPSConnection, AWSHTTPConnection +except ImportError: + try: + import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + _Boto3VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection + _cpoolBoto3HTTPConnection = cpool.HTTPConnection + _cpoolBoto3HTTPSConnection = cpool.HTTPSConnection +else: + _Boto3VerifiedHTTPSConnection = AWSHTTPSConnection + _cpoolBoto3HTTPConnection = AWSHTTPConnection + _cpoolBoto3HTTPSConnection = AWSHTTPSConnection + +cpool = None +# Try to save the original types for urllib3 +try: + import urllib3.connectionpool as cpool +except ImportError: # pragma: no cover + pass +else: + _VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection + _cpoolHTTPConnection = cpool.HTTPConnection + _cpoolHTTPSConnection = cpool.HTTPSConnection + +# Try to save the original types for requests +try: + if not cpool: + import requests.packages.urllib3.connectionpool as cpool +except ImportError: # pragma: no cover + pass +else: + _VerifiedHTTPSConnection = cpool.VerifiedHTTPSConnection + _cpoolHTTPConnection = cpool.HTTPConnection + _cpoolHTTPSConnection = cpool.HTTPSConnection + +# Try to save the original types for httplib2 +try: + import httplib2 +except ImportError: # pragma: no cover + pass +else: + _HTTPConnectionWithTimeout = httplib2.HTTPConnectionWithTimeout + _HTTPSConnectionWithTimeout = httplib2.HTTPSConnectionWithTimeout + _SCHEME_TO_CONNECTION = httplib2.SCHEME_TO_CONNECTION + +# Try to save the original types for boto +try: + import boto.https_connection +except ImportError: # pragma: no cover + pass +else: + _CertValidatingHTTPSConnection = boto.https_connection.CertValidatingHTTPSConnection + +# Try to save the original types for Tornado +try: + import tornado.simple_httpclient +except ImportError: # pragma: no cover + pass +else: + _SimpleAsyncHTTPClient_fetch_impl = tornado.simple_httpclient.SimpleAsyncHTTPClient.fetch_impl + +try: + import tornado.curl_httpclient +except ImportError: # pragma: no cover + pass +else: + _CurlAsyncHTTPClient_fetch_impl = tornado.curl_httpclient.CurlAsyncHTTPClient.fetch_impl + +try: + import aiohttp.client +except ImportError: # pragma: no cover + pass +else: + _AiohttpClientSessionRequest = aiohttp.client.ClientSession._request + + +class CassettePatcherBuilder(object): + def _build_patchers_from_mock_triples_decorator(function): + @functools.wraps(function) + def wrapped(self, *args, **kwargs): + return self._build_patchers_from_mock_triples(function(self, *args, **kwargs)) + + return wrapped + + def __init__(self, cassette): + self._cassette = cassette + self._class_to_cassette_subclass = {} + + def build(self): + return itertools.chain( + self._httplib(), + self._requests(), + self._boto3(), + self._urllib3(), + self._httplib2(), + self._boto(), + self._tornado(), + self._aiohttp(), + self._build_patchers_from_mock_triples(self._cassette.custom_patches), + ) + + def _build_patchers_from_mock_triples(self, mock_triples): + for args in mock_triples: + patcher = self._build_patcher(*args) + if patcher: + yield patcher + + def _build_patcher(self, obj, patched_attribute, replacement_class): + if not hasattr(obj, patched_attribute): + return + + return mock.patch.object( + obj, patched_attribute, self._recursively_apply_get_cassette_subclass(replacement_class) + ) + + def _recursively_apply_get_cassette_subclass(self, replacement_dict_or_obj): + """One of the subtleties of this class is that it does not directly + replace HTTPSConnection with `VCRRequestsHTTPSConnection`, but a + subclass of the aforementioned class that has the `cassette` + class attribute assigned to `self._cassette`. This behavior is + necessary to properly support nested cassette contexts. + + This function exists to ensure that we use the same class + object (reference) to patch everything that replaces + VCRRequestHTTP[S]Connection, but that we can talk about + patching them with the raw references instead, and without + worrying about exactly where the subclass with the relevant + value for `cassette` is first created. + + The function is recursive because it looks in to dictionaries + and replaces class values at any depth with the subclass + described in the previous paragraph. + """ + if isinstance(replacement_dict_or_obj, dict): + for key, replacement_obj in replacement_dict_or_obj.items(): + replacement_obj = self._recursively_apply_get_cassette_subclass(replacement_obj) + replacement_dict_or_obj[key] = replacement_obj + return replacement_dict_or_obj + if hasattr(replacement_dict_or_obj, "cassette"): + replacement_dict_or_obj = self._get_cassette_subclass(replacement_dict_or_obj) + return replacement_dict_or_obj + + def _get_cassette_subclass(self, klass): + if klass.cassette is not None: + return klass + if klass not in self._class_to_cassette_subclass: + subclass = self._build_cassette_subclass(klass) + self._class_to_cassette_subclass[klass] = subclass + return self._class_to_cassette_subclass[klass] + + def _build_cassette_subclass(self, base_class): + bases = (base_class,) + if not issubclass(base_class, object): # Check for old style class + bases += (object,) + return type( + "{}{}".format(base_class.__name__, self._cassette._path), bases, dict(cassette=self._cassette) + ) + + @_build_patchers_from_mock_triples_decorator + def _httplib(self): + yield httplib, "HTTPConnection", VCRHTTPConnection + yield httplib, "HTTPSConnection", VCRHTTPSConnection + + def _requests(self): + try: + from .stubs import requests_stubs + except ImportError: # pragma: no cover + return () + return self._urllib3_patchers(cpool, requests_stubs) + + @_build_patchers_from_mock_triples_decorator + def _boto3(self): + + try: + # botocore using awsrequest + import botocore.awsrequest as cpool + except ImportError: # pragma: no cover + try: + # botocore using vendored requests + import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + from .stubs import boto3_stubs + + yield self._urllib3_patchers(cpool, boto3_stubs) + else: + from .stubs import boto3_stubs + + log.debug("Patching boto3 cpool with %s", cpool) + yield cpool.AWSHTTPConnectionPool, "ConnectionCls", boto3_stubs.VCRRequestsHTTPConnection + yield cpool.AWSHTTPSConnectionPool, "ConnectionCls", boto3_stubs.VCRRequestsHTTPSConnection + + def _patched_get_conn(self, connection_pool_class, connection_class_getter): + get_conn = connection_pool_class._get_conn + + @functools.wraps(get_conn) + def patched_get_conn(pool, timeout=None): + connection = get_conn(pool, timeout) + connection_class = ( + pool.ConnectionCls if hasattr(pool, "ConnectionCls") else connection_class_getter() + ) + # We need to make sure that we are actually providing a + # patched version of the connection class. This might not + # always be the case because the pool keeps previously + # used connections (which might actually be of a different + # class) around. This while loop will terminate because + # eventually the pool will run out of connections. + while not isinstance(connection, connection_class): + connection = get_conn(pool, timeout) + return connection + + return patched_get_conn + + def _patched_new_conn(self, connection_pool_class, connection_remover): + new_conn = connection_pool_class._new_conn + + @functools.wraps(new_conn) + def patched_new_conn(pool): + new_connection = new_conn(pool) + connection_remover.add_connection_to_pool_entry(pool, new_connection) + return new_connection + + return patched_new_conn + + def _urllib3(self): + try: + import urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + return () + from .stubs import urllib3_stubs + + return self._urllib3_patchers(cpool, urllib3_stubs) + + @_build_patchers_from_mock_triples_decorator + def _httplib2(self): + try: + import httplib2 as cpool + except ImportError: # pragma: no cover + pass + else: + from .stubs.httplib2_stubs import VCRHTTPConnectionWithTimeout + from .stubs.httplib2_stubs import VCRHTTPSConnectionWithTimeout + + yield cpool, "HTTPConnectionWithTimeout", VCRHTTPConnectionWithTimeout + yield cpool, "HTTPSConnectionWithTimeout", VCRHTTPSConnectionWithTimeout + yield cpool, "SCHEME_TO_CONNECTION", { + "http": VCRHTTPConnectionWithTimeout, + "https": VCRHTTPSConnectionWithTimeout, + } + + @_build_patchers_from_mock_triples_decorator + def _boto(self): + try: + import boto.https_connection as cpool + except ImportError: # pragma: no cover + pass + else: + from .stubs.boto_stubs import VCRCertValidatingHTTPSConnection + + yield cpool, "CertValidatingHTTPSConnection", VCRCertValidatingHTTPSConnection + + @_build_patchers_from_mock_triples_decorator + def _tornado(self): + try: + import tornado.simple_httpclient as simple + except ImportError: # pragma: no cover + pass + else: + from .stubs.tornado_stubs import vcr_fetch_impl + + new_fetch_impl = vcr_fetch_impl(self._cassette, _SimpleAsyncHTTPClient_fetch_impl) + yield simple.SimpleAsyncHTTPClient, "fetch_impl", new_fetch_impl + try: + import tornado.curl_httpclient as curl + except ImportError: # pragma: no cover + pass + else: + from .stubs.tornado_stubs import vcr_fetch_impl + + new_fetch_impl = vcr_fetch_impl(self._cassette, _CurlAsyncHTTPClient_fetch_impl) + yield curl.CurlAsyncHTTPClient, "fetch_impl", new_fetch_impl + + @_build_patchers_from_mock_triples_decorator + def _aiohttp(self): + try: + import aiohttp.client as client + except ImportError: # pragma: no cover + pass + else: + from .stubs.aiohttp_stubs import vcr_request + + new_request = vcr_request(self._cassette, _AiohttpClientSessionRequest) + yield client.ClientSession, "_request", new_request + + def _urllib3_patchers(self, cpool, stubs): + http_connection_remover = ConnectionRemover( + self._get_cassette_subclass(stubs.VCRRequestsHTTPConnection) + ) + https_connection_remover = ConnectionRemover( + self._get_cassette_subclass(stubs.VCRRequestsHTTPSConnection) + ) + mock_triples = ( + (cpool, "VerifiedHTTPSConnection", stubs.VCRRequestsHTTPSConnection), + (cpool, "HTTPConnection", stubs.VCRRequestsHTTPConnection), + (cpool, "HTTPSConnection", stubs.VCRRequestsHTTPSConnection), + (cpool, "is_connection_dropped", mock.Mock(return_value=False)), # Needed on Windows only + (cpool.HTTPConnectionPool, "ConnectionCls", stubs.VCRRequestsHTTPConnection), + (cpool.HTTPSConnectionPool, "ConnectionCls", stubs.VCRRequestsHTTPSConnection), + ) + # These handle making sure that sessions only use the + # connections of the appropriate type. + mock_triples += ( + ( + cpool.HTTPConnectionPool, + "_get_conn", + self._patched_get_conn(cpool.HTTPConnectionPool, lambda: cpool.HTTPConnection), + ), + ( + cpool.HTTPSConnectionPool, + "_get_conn", + self._patched_get_conn(cpool.HTTPSConnectionPool, lambda: cpool.HTTPSConnection), + ), + ( + cpool.HTTPConnectionPool, + "_new_conn", + self._patched_new_conn(cpool.HTTPConnectionPool, http_connection_remover), + ), + ( + cpool.HTTPSConnectionPool, + "_new_conn", + self._patched_new_conn(cpool.HTTPSConnectionPool, https_connection_remover), + ), + ) + + return itertools.chain( + self._build_patchers_from_mock_triples(mock_triples), + (http_connection_remover, https_connection_remover), + ) + + +class ConnectionRemover(object): + def __init__(self, connection_class): + self._connection_class = connection_class + self._connection_pool_to_connections = {} + + def add_connection_to_pool_entry(self, pool, connection): + if isinstance(connection, self._connection_class): + self._connection_pool_to_connections.setdefault(pool, set()).add(connection) + + def remove_connection_to_pool_entry(self, pool, connection): + if isinstance(connection, self._connection_class): + self._connection_pool_to_connections[self._connection_class].remove(connection) + + def __enter__(self): + return self + + def __exit__(self, *args): + for pool, connections in self._connection_pool_to_connections.items(): + readd_connections = [] + while pool.pool and not pool.pool.empty() and connections: + connection = pool.pool.get() + if isinstance(connection, self._connection_class): + connections.remove(connection) + else: + readd_connections.append(connection) + for connection in readd_connections: + pool._put_conn(connection) + + +def reset_patchers(): + yield mock.patch.object(httplib, "HTTPConnection", _HTTPConnection) + yield mock.patch.object(httplib, "HTTPSConnection", _HTTPSConnection) + + try: + import requests + + if requests.__build__ < 0x021603: + # Avoid double unmock if requests 2.16.3 + # First, this is pointless, requests.packages.urllib3 *IS* urllib3 (see packages.py) + # Second, this is unmocking twice the same classes with different namespaces + # and is creating weird issues and bugs: + # > AssertionError: assert + # > is + # This assert should work!!! + # Note that this also means that now, requests.packages is never imported + # if requests 2.16.3 or greater is used with VCRPy. + import requests.packages.urllib3.connectionpool as cpool + else: + raise ImportError("Skip requests not vendored anymore") + except ImportError: # pragma: no cover + pass + else: + # unpatch requests v1.x + yield mock.patch.object(cpool, "VerifiedHTTPSConnection", _VerifiedHTTPSConnection) + yield mock.patch.object(cpool, "HTTPConnection", _cpoolHTTPConnection) + # unpatch requests v2.x + if hasattr(cpool.HTTPConnectionPool, "ConnectionCls"): + yield mock.patch.object(cpool.HTTPConnectionPool, "ConnectionCls", _cpoolHTTPConnection) + yield mock.patch.object(cpool.HTTPSConnectionPool, "ConnectionCls", _cpoolHTTPSConnection) + + if hasattr(cpool, "HTTPSConnection"): + yield mock.patch.object(cpool, "HTTPSConnection", _cpoolHTTPSConnection) + + try: + import urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + yield mock.patch.object(cpool, "VerifiedHTTPSConnection", _VerifiedHTTPSConnection) + yield mock.patch.object(cpool, "HTTPConnection", _cpoolHTTPConnection) + yield mock.patch.object(cpool, "HTTPSConnection", _cpoolHTTPSConnection) + if hasattr(cpool.HTTPConnectionPool, "ConnectionCls"): + yield mock.patch.object(cpool.HTTPConnectionPool, "ConnectionCls", _cpoolHTTPConnection) + yield mock.patch.object(cpool.HTTPSConnectionPool, "ConnectionCls", _cpoolHTTPSConnection) + + try: + # unpatch botocore with awsrequest + import botocore.awsrequest as cpool + except ImportError: # pragma: no cover + try: + # unpatch botocore with vendored requests + import botocore.vendored.requests.packages.urllib3.connectionpool as cpool + except ImportError: # pragma: no cover + pass + else: + # unpatch requests v1.x + yield mock.patch.object(cpool, "VerifiedHTTPSConnection", _Boto3VerifiedHTTPSConnection) + yield mock.patch.object(cpool, "HTTPConnection", _cpoolBoto3HTTPConnection) + # unpatch requests v2.x + if hasattr(cpool.HTTPConnectionPool, "ConnectionCls"): + yield mock.patch.object(cpool.HTTPConnectionPool, "ConnectionCls", _cpoolBoto3HTTPConnection) + yield mock.patch.object( + cpool.HTTPSConnectionPool, "ConnectionCls", _cpoolBoto3HTTPSConnection + ) + + if hasattr(cpool, "HTTPSConnection"): + yield mock.patch.object(cpool, "HTTPSConnection", _cpoolBoto3HTTPSConnection) + else: + if hasattr(cpool.AWSHTTPConnectionPool, "ConnectionCls"): + yield mock.patch.object(cpool.AWSHTTPConnectionPool, "ConnectionCls", _cpoolBoto3HTTPConnection) + yield mock.patch.object(cpool.AWSHTTPSConnectionPool, "ConnectionCls", _cpoolBoto3HTTPSConnection) + + if hasattr(cpool, "AWSHTTPSConnection"): + yield mock.patch.object(cpool, "AWSHTTPSConnection", _cpoolBoto3HTTPSConnection) + + try: + import httplib2 as cpool + except ImportError: # pragma: no cover + pass + else: + yield mock.patch.object(cpool, "HTTPConnectionWithTimeout", _HTTPConnectionWithTimeout) + yield mock.patch.object(cpool, "HTTPSConnectionWithTimeout", _HTTPSConnectionWithTimeout) + yield mock.patch.object(cpool, "SCHEME_TO_CONNECTION", _SCHEME_TO_CONNECTION) + + try: + import boto.https_connection as cpool + except ImportError: # pragma: no cover + pass + else: + yield mock.patch.object(cpool, "CertValidatingHTTPSConnection", _CertValidatingHTTPSConnection) + + try: + import tornado.simple_httpclient as simple + except ImportError: # pragma: no cover + pass + else: + yield mock.patch.object(simple.SimpleAsyncHTTPClient, "fetch_impl", _SimpleAsyncHTTPClient_fetch_impl) + try: + import tornado.curl_httpclient as curl + except ImportError: # pragma: no cover + pass + else: + yield mock.patch.object(curl.CurlAsyncHTTPClient, "fetch_impl", _CurlAsyncHTTPClient_fetch_impl) + + +@contextlib.contextmanager +def force_reset(): + with contextlib.ExitStack() as exit_stack: + for patcher in reset_patchers(): + exit_stack.enter_context(patcher) + yield diff --git a/tools/vcrpy/vcr/persisters/__init__.py b/tools/vcrpy/vcr/persisters/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tools/vcrpy/vcr/persisters/filesystem.py b/tools/vcrpy/vcr/persisters/filesystem.py new file mode 100644 index 000000000000..e8d82709b8cb --- /dev/null +++ b/tools/vcrpy/vcr/persisters/filesystem.py @@ -0,0 +1,25 @@ +# .. _persister_example: + +import os +from ..serialize import serialize, deserialize + + +class FilesystemPersister(object): + @classmethod + def load_cassette(cls, cassette_path, serializer): + try: + with open(cassette_path) as f: + cassette_content = f.read() + except IOError: + raise ValueError("Cassette not found.") + cassette = deserialize(cassette_content, serializer) + return cassette + + @staticmethod + def save_cassette(cassette_path, cassette_dict, serializer): + data = serialize(cassette_dict, serializer) + dirname, filename = os.path.split(cassette_path) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + with open(cassette_path, "w") as f: + f.write(data) diff --git a/tools/vcrpy/vcr/request.py b/tools/vcrpy/vcr/request.py new file mode 100644 index 000000000000..d96517ff41ef --- /dev/null +++ b/tools/vcrpy/vcr/request.py @@ -0,0 +1,139 @@ +import warnings +from six import BytesIO, text_type +from six.moves.urllib.parse import urlparse, parse_qsl +from .util import CaseInsensitiveDict +import logging + +log = logging.getLogger(__name__) + + +class Request(object): + """ + VCR's representation of a request. + """ + + def __init__(self, method, uri, body, headers): + self.method = method + self.uri = uri + self._was_file = hasattr(body, "read") + if self._was_file: + self.body = body.read() + else: + self.body = body + self.headers = headers + log.debug("Invoking Request %s", self.uri) + + @property + def headers(self): + return self._headers + + @headers.setter + def headers(self, value): + if not isinstance(value, HeadersDict): + value = HeadersDict(value) + self._headers = value + + @property + def body(self): + return BytesIO(self._body) if self._was_file else self._body + + @body.setter + def body(self, value): + if isinstance(value, text_type): + value = value.encode("utf-8") + self._body = value + + def add_header(self, key, value): + warnings.warn( + "Request.add_header is deprecated. " "Please assign to request.headers instead.", + DeprecationWarning, + ) + self.headers[key] = value + + @property + def scheme(self): + return urlparse(self.uri).scheme + + @property + def host(self): + return urlparse(self.uri).hostname + + @property + def port(self): + parse_uri = urlparse(self.uri) + port = parse_uri.port + if port is None: + try: + port = {"https": 443, "http": 80}[parse_uri.scheme] + except KeyError: + pass + return port + + @property + def path(self): + return urlparse(self.uri).path + + @property + def query(self): + q = urlparse(self.uri).query + return sorted(parse_qsl(q)) + + # alias for backwards compatibility + @property + def url(self): + return self.uri + + # alias for backwards compatibility + @property + def protocol(self): + return self.scheme + + def __str__(self): + return "".format(self.method, self.uri) + + def __repr__(self): + return self.__str__() + + def _to_dict(self): + return { + "method": self.method, + "uri": self.uri, + "body": self.body, + "headers": {k: [v] for k, v in self.headers.items()}, + } + + @classmethod + def _from_dict(cls, dct): + return Request(**dct) + + +class HeadersDict(CaseInsensitiveDict): + """ + There is a weird quirk in HTTP. You can send the same header twice. For + this reason, headers are represented by a dict, with lists as the values. + However, it appears that HTTPlib is completely incapable of sending the + same header twice. This puts me in a weird position: I want to be able to + accurately represent HTTP headers in cassettes, but I don't want the extra + step of always having to do [0] in the general case, i.e. + request.headers['key'][0] + + In addition, some servers sometimes send the same header more than once, + and httplib *can* deal with this situation. + + Furthermore, I wanted to keep the request and response cassette format as + similar as possible. + + For this reason, in cassettes I keep a dict with lists as keys, but once + deserialized into VCR, I keep them as plain, naked dicts. + """ + + def __setitem__(self, key, value): + if isinstance(value, (tuple, list)): + value = value[0] + + # Preserve the case from the first time this key was set. + old = self._store.get(key.lower()) + if old: + key = old[0] + + super(HeadersDict, self).__setitem__(key, value) diff --git a/tools/vcrpy/vcr/serialize.py b/tools/vcrpy/vcr/serialize.py new file mode 100644 index 000000000000..89ffa3571946 --- /dev/null +++ b/tools/vcrpy/vcr/serialize.py @@ -0,0 +1,58 @@ +from vcr.serializers import compat +from vcr.request import Request +import yaml + +# version 1 cassettes started with VCR 1.0.x. +# Before 1.0.x, there was no versioning. +CASSETTE_FORMAT_VERSION = 1 + +""" +Just a general note on the serialization philosophy here: +I prefer cassettes to be human-readable if possible. Yaml serializes +bytestrings to !!binary, which isn't readable, so I would like to serialize to +strings and from strings, which yaml will encode as utf-8 automatically. +All the internal HTTP stuff expects bytestrings, so this whole serialization +process feels backwards. + +Serializing: bytestring -> string (yaml persists to utf-8) +Deserializing: string (yaml converts from utf-8) -> bytestring +""" + + +def _looks_like_an_old_cassette(data): + return isinstance(data, list) and len(data) and "request" in data[0] + + +def _warn_about_old_cassette_format(): + raise ValueError( + "Your cassette files were generated in an older version " + "of VCR. Delete your cassettes or run the migration script." + "See http://git.io/mHhLBg for more details." + ) + + +def deserialize(cassette_string, serializer): + try: + data = serializer.deserialize(cassette_string) + # Old cassettes used to use yaml object thingy so I have to + # check for some fairly stupid exceptions here + except (ImportError, yaml.constructor.ConstructorError): + _warn_about_old_cassette_format() + if _looks_like_an_old_cassette(data): + _warn_about_old_cassette_format() + + requests = [Request._from_dict(r["request"]) for r in data["interactions"]] + responses = [compat.convert_to_bytes(r["response"]) for r in data["interactions"]] + return requests, responses + + +def serialize(cassette_dict, serializer): + interactions = [ + { + "request": compat.convert_to_unicode(request._to_dict()), + "response": compat.convert_to_unicode(response), + } + for request, response in zip(cassette_dict["requests"], cassette_dict["responses"]) + ] + data = {"version": CASSETTE_FORMAT_VERSION, "interactions": interactions} + return serializer.serialize(data) diff --git a/tools/vcrpy/vcr/serializers/__init__.py b/tools/vcrpy/vcr/serializers/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tools/vcrpy/vcr/serializers/compat.py b/tools/vcrpy/vcr/serializers/compat.py new file mode 100644 index 000000000000..bbc201c4bb23 --- /dev/null +++ b/tools/vcrpy/vcr/serializers/compat.py @@ -0,0 +1,77 @@ +import six + + +def convert_to_bytes(resp): + resp = convert_body_to_bytes(resp) + return resp + + +def convert_to_unicode(resp): + resp = convert_body_to_unicode(resp) + return resp + + +def convert_body_to_bytes(resp): + """ + If the request body is a string, encode it to bytes (for python3 support) + + By default yaml serializes to utf-8 encoded bytestrings. + When this cassette is loaded by python3, it's automatically decoded + into unicode strings. This makes sure that it stays a bytestring, since + that's what all the internal httplib machinery is expecting. + + For more info on py3 yaml: + http://pyyaml.org/wiki/PyYAMLDocumentation#Python3support + """ + try: + if resp["body"]["string"] is not None and not isinstance(resp["body"]["string"], six.binary_type): + resp["body"]["string"] = resp["body"]["string"].encode("utf-8") + except (KeyError, TypeError, UnicodeEncodeError): + # The thing we were converting either wasn't a dictionary or didn't + # have the keys we were expecting. Some of the tests just serialize + # and deserialize a string. + + # Also, sometimes the thing actually is binary, so if you can't encode + # it, just give up. + pass + return resp + + +def _convert_string_to_unicode(string): + """ + If the string is bytes, decode it to a string (for python3 support) + """ + result = string + + try: + if string is not None and not isinstance(string, six.text_type): + result = string.decode("utf-8") + except (TypeError, UnicodeDecodeError, AttributeError): + # Sometimes the string actually is binary or StringIO object, + # so if you can't decode it, just give up. + pass + + return result + + +def convert_body_to_unicode(resp): + """ + If the request or responses body is bytes, decode it to a string + (for python3 support) + """ + if type(resp) is not dict: + # Some of the tests just serialize and deserialize a string. + return _convert_string_to_unicode(resp) + else: + body = resp.get("body") + + if body is not None: + try: + body["string"] = _convert_string_to_unicode(body["string"]) + except (KeyError, TypeError, AttributeError): + # The thing we were converting either wasn't a dictionary or + # didn't have the keys we were expecting. + # For example request object has no 'string' key. + resp["body"] = _convert_string_to_unicode(body) + + return resp diff --git a/tools/vcrpy/vcr/serializers/jsonserializer.py b/tools/vcrpy/vcr/serializers/jsonserializer.py new file mode 100644 index 000000000000..e5ff85a9d9db --- /dev/null +++ b/tools/vcrpy/vcr/serializers/jsonserializer.py @@ -0,0 +1,29 @@ +try: + import simplejson as json +except ImportError: + import json + + +def deserialize(cassette_string): + return json.loads(cassette_string) + + +def serialize(cassette_dict): + error_message = ( + "Does this HTTP interaction contain binary data? " + "If so, use a different serializer (like the yaml serializer) " + "for this request?" + ) + + try: + return json.dumps(cassette_dict, indent=4) + except UnicodeDecodeError as original: # py2 + raise UnicodeDecodeError( + original.encoding, + b"Error serializing cassette to JSON", + original.start, + original.end, + original.args[-1] + error_message, + ) + except TypeError: # py3 + raise TypeError(error_message) diff --git a/tools/vcrpy/vcr/serializers/yamlserializer.py b/tools/vcrpy/vcr/serializers/yamlserializer.py new file mode 100644 index 000000000000..0d6afc96c02c --- /dev/null +++ b/tools/vcrpy/vcr/serializers/yamlserializer.py @@ -0,0 +1,15 @@ +import yaml + +# Use the libYAML versions if possible +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + + +def deserialize(cassette_string): + return yaml.load(cassette_string, Loader=Loader) + + +def serialize(cassette_dict): + return yaml.dump(cassette_dict, Dumper=Dumper) diff --git a/tools/vcrpy/vcr/stubs/__init__.py b/tools/vcrpy/vcr/stubs/__init__.py new file mode 100644 index 000000000000..37ab137ce477 --- /dev/null +++ b/tools/vcrpy/vcr/stubs/__init__.py @@ -0,0 +1,363 @@ +"""Stubs for patching HTTP and HTTPS requests""" + +import logging +import six +from six.moves.http_client import HTTPConnection, HTTPSConnection, HTTPResponse +from six import BytesIO +from vcr.request import Request +from vcr.errors import CannotOverwriteExistingCassetteException +from . import compat + +log = logging.getLogger(__name__) + + +class VCRFakeSocket(object): + """ + A socket that doesn't do anything! + Used when playing back cassettes, when there + is no actual open socket. + """ + + def close(self): + pass + + def settimeout(self, *args, **kwargs): + pass + + def fileno(self): + """ + This is kinda crappy. requests will watch + this descriptor and make sure it's not closed. + Return file descriptor 0 since that's stdin. + """ + return 0 # wonder how bad this is.... + + +def parse_headers(header_list): + """ + Convert headers from our serialized dict with lists for keys to a + HTTPMessage + """ + header_string = b"" + for key, values in header_list.items(): + for v in values: + header_string += key.encode("utf-8") + b":" + v.encode("utf-8") + b"\r\n" + return compat.get_httpmessage(header_string) + + +def serialize_headers(response): + out = {} + for key, values in compat.get_headers(response.msg): + out.setdefault(key, []) + out[key].extend(values) + return out + + +class VCRHTTPResponse(HTTPResponse): + """ + Stub response class that gets returned instead of a HTTPResponse + """ + + def __init__(self, recorded_response): + self.fp = None + self.recorded_response = recorded_response + self.reason = recorded_response["status"]["message"] + self.status = self.code = recorded_response["status"]["code"] + self.version = None + self._content = BytesIO(self.recorded_response["body"]["string"]) + self._closed = False + + headers = self.recorded_response["headers"] + # Since we are loading a response that has already been serialized, our + # response is no longer chunked. That means we don't want any + # libraries trying to process a chunked response. By removing the + # transfer-encoding: chunked header, this should cause the downstream + # libraries to process this as a non-chunked response. + te_key = [h for h in headers.keys() if h.upper() == "TRANSFER-ENCODING"] + if te_key: + del headers[te_key[0]] + self.headers = self.msg = parse_headers(headers) + + self.length = compat.get_header(self.msg, "content-length") or None + + @property + def closed(self): + # in python3, I can't change the value of self.closed. So I' + # twiddling self._closed and using this property to shadow the real + # self.closed from the superclas + return self._closed + + def read(self, *args, **kwargs): + return self._content.read(*args, **kwargs) + + def readall(self): + return self._content.readall() + + def readinto(self, *args, **kwargs): + return self._content.readinto(*args, **kwargs) + + def readline(self, *args, **kwargs): + return self._content.readline(*args, **kwargs) + + def readlines(self, *args, **kwargs): + return self._content.readlines(*args, **kwargs) + + def seekable(self): + return self._content.seekable() + + def tell(self): + return self._content.tell() + + def isatty(self): + return self._content.isatty() + + def seek(self, *args, **kwargs): + return self._content.seek(*args, **kwargs) + + def close(self): + self._closed = True + return True + + def getcode(self): + return self.status + + def isclosed(self): + return self.closed + + def info(self): + return parse_headers(self.recorded_response["headers"]) + + def getheaders(self): + message = parse_headers(self.recorded_response["headers"]) + return list(compat.get_header_items(message)) + + def getheader(self, header, default=None): + values = [v for (k, v) in self.getheaders() if k.lower() == header.lower()] + + if values: + return ", ".join(values) + else: + return default + + def readable(self): + return self._content.readable() + + +class VCRConnection(object): + # A reference to the cassette that's currently being patched in + cassette = None + + def _port_postfix(self): + """ + Returns empty string for the default port and ':port' otherwise + """ + port = self.real_connection.port + default_port = {"https": 443, "http": 80}[self._protocol] + return ":{}".format(port) if port != default_port else "" + + def _uri(self, url): + """Returns request absolute URI""" + if url and not url.startswith("/"): + # Then this must be a proxy request. + return url + uri = "{}://{}{}{}".format(self._protocol, self.real_connection.host, self._port_postfix(), url) + log.debug("Absolute URI: %s", uri) + return uri + + def _url(self, uri): + """Returns request selector url from absolute URI""" + prefix = "{}://{}{}".format(self._protocol, self.real_connection.host, self._port_postfix()) + return uri.replace(prefix, "", 1) + + def request(self, method, url, body=None, headers=None, *args, **kwargs): + """Persist the request metadata in self._vcr_request""" + self._vcr_request = Request(method=method, uri=self._uri(url), body=body, headers=headers or {}) + log.debug("Got {}".format(self._vcr_request)) + + # Note: The request may not actually be finished at this point, so + # I'm not sending the actual request until getresponse(). This + # allows me to compare the entire length of the response to see if it + # exists in the cassette. + + self._sock = VCRFakeSocket() + + def putrequest(self, method, url, *args, **kwargs): + """ + httplib gives you more than one way to do it. This is a way + to start building up a request. Usually followed by a bunch + of putheader() calls. + """ + self._vcr_request = Request(method=method, uri=self._uri(url), body="", headers={}) + log.debug("Got {}".format(self._vcr_request)) + + def putheader(self, header, *values): + self._vcr_request.headers[header] = values + + def send(self, data): + """ + This method is called after request(), to add additional data to the + body of the request. So if that happens, let's just append the data + onto the most recent request in the cassette. + """ + self._vcr_request.body = self._vcr_request.body + data if self._vcr_request.body else data + + def close(self): + # Note: the real connection will only close if it's open, so + # no need to check that here. + self.real_connection.close() + + def endheaders(self, message_body=None): + """ + Normally, this would actually send the request to the server. + We are not sending the request until getting the response, + so bypass this part and just append the message body, if any. + """ + if message_body is not None: + self._vcr_request.body = message_body + + def getresponse(self, _=False, **kwargs): + """Retrieve the response""" + # Check to see if the cassette has a response for this request. If so, + # then return it + if self.cassette.can_play_response_for(self._vcr_request): + log.info("Playing response for {} from cassette".format(self._vcr_request)) + response = self.cassette.play_response(self._vcr_request) + return VCRHTTPResponse(response) + else: + if self.cassette.write_protected and self.cassette.filter_request(self._vcr_request): + raise CannotOverwriteExistingCassetteException( + cassette=self.cassette, failed_request=self._vcr_request + ) + + # Otherwise, we should send the request, then get the response + # and return it. + + log.info("{} not in cassette, sending to real server".format(self._vcr_request)) + # This is imported here to avoid circular import. + # TODO(@IvanMalison): Refactor to allow normal import. + from vcr.patch import force_reset + + with force_reset(): + self.real_connection.request( + method=self._vcr_request.method, + url=self._url(self._vcr_request.uri), + body=self._vcr_request.body, + headers=self._vcr_request.headers, + ) + + # get the response + response = self.real_connection.getresponse() + + # put the response into the cassette + response = { + "status": {"code": response.status, "message": response.reason}, + "headers": serialize_headers(response), + "body": {"string": response.read()}, + } + self.cassette.append(self._vcr_request, response) + return VCRHTTPResponse(response) + + def set_debuglevel(self, *args, **kwargs): + self.real_connection.set_debuglevel(*args, **kwargs) + + def connect(self, *args, **kwargs): + """ + httplib2 uses this. Connects to the server I'm assuming. + + Only pass to the baseclass if we don't have a recorded response + and are not write-protected. + """ + + if hasattr(self, "_vcr_request") and self.cassette.can_play_response_for(self._vcr_request): + # We already have a response we are going to play, don't + # actually connect + return + + if self.cassette.write_protected: + # Cassette is write-protected, don't actually connect + return + + from vcr.patch import force_reset + + with force_reset(): + return self.real_connection.connect(*args, **kwargs) + + self._sock = VCRFakeSocket() + + @property + def sock(self): + if self.real_connection.sock: + return self.real_connection.sock + return self._sock + + @sock.setter + def sock(self, value): + if self.real_connection.sock: + self.real_connection.sock = value + + def __init__(self, *args, **kwargs): + if six.PY3: + kwargs.pop("strict", None) # apparently this is gone in py3 + + # need to temporarily reset here because the real connection + # inherits from the thing that we are mocking out. Take out + # the reset if you want to see what I mean :) + from vcr.patch import force_reset + + with force_reset(): + self.real_connection = self._baseclass(*args, **kwargs) + + self._sock = None + + def __setattr__(self, name, value): + """ + We need to define this because any attributes that are set on the + VCRConnection need to be propogated to the real connection. + + For example, urllib3 will set certain attributes on the connection, + such as 'ssl_version'. These attributes need to get set on the real + connection to have the correct and expected behavior. + + TODO: Separately setting the attribute on the two instances is not + ideal. We should switch to a proxying implementation. + """ + try: + setattr(self.real_connection, name, value) + except AttributeError: + # raised if real_connection has not been set yet, such as when + # we're setting the real_connection itself for the first time + pass + + super(VCRConnection, self).__setattr__(name, value) + + def __getattr__(self, name): + """ + Send requests for weird attributes up to the real connection + (counterpart to __setattr above) + """ + if self.__dict__.get("real_connection"): + # check in case real_connection has not been set yet, such as when + # we're setting the real_connection itself for the first time + return getattr(self.real_connection, name) + + return super(VCRConnection, self).__getattr__(name) + + +for k, v in HTTPConnection.__dict__.items(): + if isinstance(v, staticmethod): + setattr(VCRConnection, k, v) + + +class VCRHTTPConnection(VCRConnection): + """A Mocked class for HTTP requests""" + + _baseclass = HTTPConnection + _protocol = "http" + + +class VCRHTTPSConnection(VCRConnection): + """A Mocked class for HTTPS requests""" + + _baseclass = HTTPSConnection + _protocol = "https" + is_verified = True diff --git a/tools/vcrpy/vcr/stubs/aiohttp_stubs/__init__.py b/tools/vcrpy/vcr/stubs/aiohttp_stubs/__init__.py new file mode 100644 index 000000000000..2301334a479c --- /dev/null +++ b/tools/vcrpy/vcr/stubs/aiohttp_stubs/__init__.py @@ -0,0 +1,209 @@ +"""Stubs for aiohttp HTTP clients""" +from __future__ import absolute_import + +import asyncio +import functools +import logging +import json + +from aiohttp import ClientConnectionError, ClientResponse, RequestInfo, streams +from multidict import CIMultiDict, CIMultiDictProxy +from yarl import URL + +from vcr.request import Request + +log = logging.getLogger(__name__) + + +class MockStream(asyncio.StreamReader, streams.AsyncStreamReaderMixin): + pass + + +class MockClientResponse(ClientResponse): + def __init__(self, method, url, request_info=None): + super().__init__( + method=method, + url=url, + writer=None, + continue100=None, + timer=None, + request_info=request_info, + traces=None, + loop=asyncio.get_event_loop(), + session=None, + ) + + async def json(self, *, encoding="utf-8", loads=json.loads, **kwargs): # NOQA: E999 + stripped = self._body.strip() + if not stripped: + return None + + return loads(stripped.decode(encoding)) + + async def text(self, encoding="utf-8", errors="strict"): + return self._body.decode(encoding, errors=errors) + + async def read(self): + return self._body + + def release(self): + pass + + @property + def content(self): + s = MockStream() + s.feed_data(self._body) + s.feed_eof() + return s + + +def build_response(vcr_request, vcr_response, history): + request_info = RequestInfo( + url=URL(vcr_request.url), + method=vcr_request.method, + headers=CIMultiDictProxy(CIMultiDict(vcr_request.headers)), + real_url=URL(vcr_request.url), + ) + response = MockClientResponse(vcr_request.method, URL(vcr_response.get("url")), request_info=request_info) + response.status = vcr_response["status"]["code"] + response._body = vcr_response["body"].get("string", b"") + response.reason = vcr_response["status"]["message"] + response._headers = CIMultiDictProxy(CIMultiDict(vcr_response["headers"])) + response._history = tuple(history) + + response.close() + return response + + +def _serialize_headers(headers): + """Serialize CIMultiDictProxy to a pickle-able dict because proxy + objects forbid pickling: + + https://github.com/aio-libs/multidict/issues/340 + """ + # Mark strings as keys so 'istr' types don't show up in + # the cassettes as comments. + return {str(k): v for k, v in headers.items()} + + +def play_responses(cassette, vcr_request): + history = [] + vcr_response = cassette.play_response(vcr_request) + response = build_response(vcr_request, vcr_response, history) + + # If we're following redirects, continue playing until we reach + # our final destination. + while 300 <= response.status <= 399: + next_url = URL(response.url).with_path(response.headers["location"]) + + # Make a stub VCR request that we can then use to look up the recorded + # VCR request saved to the cassette. This feels a little hacky and + # may have edge cases based on the headers we're providing (e.g. if + # there's a matcher that is used to filter by headers). + vcr_request = Request("GET", str(next_url), None, _serialize_headers(response.request_info.headers)) + vcr_request = cassette.find_requests_with_most_matches(vcr_request)[0][0] + + # Tack on the response we saw from the redirect into the history + # list that is added on to the final response. + history.append(response) + vcr_response = cassette.play_response(vcr_request) + response = build_response(vcr_request, vcr_response, history) + + return response + + +async def record_response(cassette, vcr_request, response): + """Record a VCR request-response chain to the cassette.""" + + try: + body = {"string": (await response.read())} + # aiohttp raises a ClientConnectionError on reads when + # there is no body. We can use this to know to not write one. + except ClientConnectionError: + body = {} + + vcr_response = { + "status": {"code": response.status, "message": response.reason}, + "headers": _serialize_headers(response.headers), + "body": body, # NOQA: E999 + "url": str(response.url), + } + + cassette.append(vcr_request, vcr_response) + + +async def record_responses(cassette, vcr_request, response): + """Because aiohttp follows redirects by default, we must support + them by default. This method is used to write individual + request-response chains that were implicitly followed to get + to the final destination. + """ + + for past_response in response.history: + aiohttp_request = past_response.request_info + + # No data because it's following a redirect. + past_request = Request( + aiohttp_request.method, + str(aiohttp_request.url), + None, + _serialize_headers(aiohttp_request.headers), + ) + await record_response(cassette, past_request, past_response) + + # If we're following redirects, then the last request-response + # we record is the one attached to the `response`. + if response.history: + aiohttp_request = response.request_info + vcr_request = Request( + aiohttp_request.method, + str(aiohttp_request.url), + None, + _serialize_headers(aiohttp_request.headers), + ) + + await record_response(cassette, vcr_request, response) + + +def vcr_request(cassette, real_request): + @functools.wraps(real_request) + async def new_request(self, method, url, **kwargs): + headers = kwargs.get("headers") + auth = kwargs.get("auth") + headers = self._prepare_headers(headers) + data = kwargs.get("data", kwargs.get("json")) + params = kwargs.get("params") + + if auth is not None: + headers["AUTHORIZATION"] = auth.encode() + + request_url = URL(url) + if params: + for k, v in params.items(): + params[k] = str(v) + request_url = URL(url).with_query(params) + + vcr_request = Request(method, str(request_url), data, headers) + + if cassette.can_play_response_for(vcr_request): + return play_responses(cassette, vcr_request) + + if cassette.write_protected and cassette.filter_request(vcr_request): + response = MockClientResponse(method, URL(url)) + response.status = 599 + msg = ( + "No match for the request {!r} was found. Can't overwrite " + "existing cassette {!r} in your current record mode {!r}." + ) + msg = msg.format(vcr_request, cassette._path, cassette.record_mode) + response._body = msg.encode() + response.close() + return response + + log.info("%s not in cassette, sending to real server", vcr_request) + + response = await real_request(self, method, url, **kwargs) # NOQA: E999 + await record_responses(cassette, vcr_request, response) + return response + + return new_request diff --git a/tools/vcrpy/vcr/stubs/boto3_stubs.py b/tools/vcrpy/vcr/stubs/boto3_stubs.py new file mode 100644 index 000000000000..1a0e4a2d2613 --- /dev/null +++ b/tools/vcrpy/vcr/stubs/boto3_stubs.py @@ -0,0 +1,44 @@ +"""Stubs for boto3""" +import six + +try: + # boto using awsrequest + from botocore.awsrequest import AWSHTTPConnection as HTTPConnection + from botocore.awsrequest import AWSHTTPSConnection as VerifiedHTTPSConnection + +except ImportError: # pragma: nocover + # boto using vendored requests + # urllib3 defines its own HTTPConnection classes, which boto3 goes ahead and assumes + # you're using. It includes some polyfills for newer features missing in older pythons. + try: + from urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection + except ImportError: # pragma: nocover + from requests.packages.urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection + +from ..stubs import VCRHTTPConnection, VCRHTTPSConnection + + +class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection): + _baseclass = HTTPConnection + + +class VCRRequestsHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection): + _baseclass = VerifiedHTTPSConnection + + def __init__(self, *args, **kwargs): + if six.PY3: + kwargs.pop("strict", None) # apparently this is gone in py3 + + # need to temporarily reset here because the real connection + # inherits from the thing that we are mocking out. Take out + # the reset if you want to see what I mean :) + from vcr.patch import force_reset + + with force_reset(): + self.real_connection = self._baseclass(*args, **kwargs) + # Make sure to set those attributes as it seems `AWSHTTPConnection` does not + # set them, making the connection to fail ! + self.real_connection.assert_hostname = kwargs.get("assert_hostname", False) + self.real_connection.cert_reqs = kwargs.get("cert_reqs", "CERT_NONE") + + self._sock = None diff --git a/tools/vcrpy/vcr/stubs/boto_stubs.py b/tools/vcrpy/vcr/stubs/boto_stubs.py new file mode 100644 index 000000000000..d43f1b5f6114 --- /dev/null +++ b/tools/vcrpy/vcr/stubs/boto_stubs.py @@ -0,0 +1,8 @@ +"""Stubs for boto""" + +from boto.https_connection import CertValidatingHTTPSConnection +from ..stubs import VCRHTTPSConnection + + +class VCRCertValidatingHTTPSConnection(VCRHTTPSConnection): + _baseclass = CertValidatingHTTPSConnection diff --git a/tools/vcrpy/vcr/stubs/compat.py b/tools/vcrpy/vcr/stubs/compat.py new file mode 100644 index 000000000000..938d651ae74d --- /dev/null +++ b/tools/vcrpy/vcr/stubs/compat.py @@ -0,0 +1,44 @@ +import six +from six import BytesIO +from six.moves.http_client import HTTPMessage + +try: + import http.client +except ImportError: + pass + + +""" +The python3 http.client api moved some stuff around, so this is an abstraction +layer that tries to cope with this move. +""" + + +def get_header(message, name): + if six.PY3: + return message.getallmatchingheaders(name) + else: + return message.getheader(name) + + +def get_header_items(message): + for (key, values) in get_headers(message): + for value in values: + yield key, value + + +def get_headers(message): + for key in set(message.keys()): + if six.PY3: + yield key, message.get_all(key) + else: + yield key, message.getheaders(key) + + +def get_httpmessage(headers): + if six.PY3: + return http.client.parse_headers(BytesIO(headers)) + msg = HTTPMessage(BytesIO(headers)) + msg.fp.seek(0) + msg.readheaders() + return msg diff --git a/tools/vcrpy/vcr/stubs/httplib2_stubs.py b/tools/vcrpy/vcr/stubs/httplib2_stubs.py new file mode 100644 index 000000000000..e79dcc9813c1 --- /dev/null +++ b/tools/vcrpy/vcr/stubs/httplib2_stubs.py @@ -0,0 +1,60 @@ +"""Stubs for httplib2""" + +from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout +from ..stubs import VCRHTTPConnection, VCRHTTPSConnection + + +class VCRHTTPConnectionWithTimeout(VCRHTTPConnection, HTTPConnectionWithTimeout): + _baseclass = HTTPConnectionWithTimeout + + def __init__(self, *args, **kwargs): + """I overrode the init because I need to clean kwargs before calling + HTTPConnection.__init__.""" + + # Delete the keyword arguments that HTTPConnection would not recognize + safe_keys = {"host", "port", "strict", "timeout", "source_address"} + unknown_keys = set(kwargs.keys()) - safe_keys + safe_kwargs = kwargs.copy() + for kw in unknown_keys: + del safe_kwargs[kw] + + self.proxy_info = kwargs.pop("proxy_info", None) + VCRHTTPConnection.__init__(self, *args, **safe_kwargs) + self.sock = self.real_connection.sock + + +class VCRHTTPSConnectionWithTimeout(VCRHTTPSConnection, HTTPSConnectionWithTimeout): + _baseclass = HTTPSConnectionWithTimeout + + def __init__(self, *args, **kwargs): + + # Delete the keyword arguments that HTTPSConnection would not recognize + safe_keys = { + "host", + "port", + "key_file", + "cert_file", + "strict", + "timeout", + "source_address", + "ca_certs", + "disable_ssl_certificate_validation", + } + unknown_keys = set(kwargs.keys()) - safe_keys + safe_kwargs = kwargs.copy() + for kw in unknown_keys: + del safe_kwargs[kw] + self.proxy_info = kwargs.pop("proxy_info", None) + if "ca_certs" not in kwargs or kwargs["ca_certs"] is None: + try: + import httplib2 + + self.ca_certs = httplib2.CA_CERTS + except ImportError: + self.ca_certs = None + else: + self.ca_certs = kwargs["ca_certs"] + + self.disable_ssl_certificate_validation = kwargs.pop("disable_ssl_certificate_validation", None) + VCRHTTPSConnection.__init__(self, *args, **safe_kwargs) + self.sock = self.real_connection.sock diff --git a/tools/vcrpy/vcr/stubs/requests_stubs.py b/tools/vcrpy/vcr/stubs/requests_stubs.py new file mode 100644 index 000000000000..e547b811b4eb --- /dev/null +++ b/tools/vcrpy/vcr/stubs/requests_stubs.py @@ -0,0 +1,19 @@ +"""Stubs for requests""" + +try: + from urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection +except ImportError: + from requests.packages.urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection + +from ..stubs import VCRHTTPConnection, VCRHTTPSConnection + +# urllib3 defines its own HTTPConnection classes, which requests goes ahead and assumes +# you're using. It includes some polyfills for newer features missing in older pythons. + + +class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection): + _baseclass = HTTPConnection + + +class VCRRequestsHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection): + _baseclass = VerifiedHTTPSConnection diff --git a/tools/vcrpy/vcr/stubs/tornado_stubs.py b/tools/vcrpy/vcr/stubs/tornado_stubs.py new file mode 100644 index 000000000000..c6ecc2efb9e1 --- /dev/null +++ b/tools/vcrpy/vcr/stubs/tornado_stubs.py @@ -0,0 +1,90 @@ +"""Stubs for tornado HTTP clients""" +from __future__ import absolute_import + +import functools +from six import BytesIO + +from tornado import httputil +from tornado.httpclient import HTTPResponse + +from vcr.errors import CannotOverwriteExistingCassetteException +from vcr.request import Request + + +def vcr_fetch_impl(cassette, real_fetch_impl): + @functools.wraps(real_fetch_impl) + def new_fetch_impl(self, request, callback): + headers = request.headers.copy() + if request.user_agent: + headers.setdefault("User-Agent", request.user_agent) + + # TODO body_producer, header_callback, and streaming_callback are not + # yet supported. + + unsupported_call = ( + getattr(request, "body_producer", None) is not None + or request.header_callback is not None + or request.streaming_callback is not None + ) + if unsupported_call: + response = HTTPResponse( + request, + 599, + error=Exception( + "The request (%s) uses AsyncHTTPClient functionality " + "that is not yet supported by VCR.py. Please make the " + "request outside a VCR.py context." % repr(request) + ), + request_time=self.io_loop.time() - request.start_time, + ) + return callback(response) + + vcr_request = Request(request.method, request.url, request.body, headers) + + if cassette.can_play_response_for(vcr_request): + vcr_response = cassette.play_response(vcr_request) + headers = httputil.HTTPHeaders() + + recorded_headers = vcr_response["headers"] + if isinstance(recorded_headers, dict): + recorded_headers = recorded_headers.items() + for k, vs in recorded_headers: + for v in vs: + headers.add(k, v) + response = HTTPResponse( + request, + code=vcr_response["status"]["code"], + reason=vcr_response["status"]["message"], + headers=headers, + buffer=BytesIO(vcr_response["body"]["string"]), + effective_url=vcr_response.get("url"), + request_time=self.io_loop.time() - request.start_time, + ) + return callback(response) + else: + if cassette.write_protected and cassette.filter_request(vcr_request): + response = HTTPResponse( + request, + 599, + error=CannotOverwriteExistingCassetteException( + cassette=cassette, failed_request=vcr_request + ), + request_time=self.io_loop.time() - request.start_time, + ) + return callback(response) + + def new_callback(response): + headers = [(k, response.headers.get_list(k)) for k in response.headers.keys()] + + vcr_response = { + "status": {"code": response.code, "message": response.reason}, + "headers": headers, + "body": {"string": response.body}, + "url": response.effective_url, + } + cassette.append(vcr_request, vcr_response) + return callback(response) + + real_fetch_impl(self, request, new_callback) + + return new_fetch_impl diff --git a/tools/vcrpy/vcr/stubs/urllib3_stubs.py b/tools/vcrpy/vcr/stubs/urllib3_stubs.py new file mode 100644 index 000000000000..56e5f95bf3a1 --- /dev/null +++ b/tools/vcrpy/vcr/stubs/urllib3_stubs.py @@ -0,0 +1,15 @@ +"""Stubs for urllib3""" + +from urllib3.connectionpool import HTTPConnection, VerifiedHTTPSConnection +from ..stubs import VCRHTTPConnection, VCRHTTPSConnection + +# urllib3 defines its own HTTPConnection classes. It includes some polyfills +# for newer features missing in older pythons. + + +class VCRRequestsHTTPConnection(VCRHTTPConnection, HTTPConnection): + _baseclass = HTTPConnection + + +class VCRRequestsHTTPSConnection(VCRHTTPSConnection, VerifiedHTTPSConnection): + _baseclass = VerifiedHTTPSConnection diff --git a/tools/vcrpy/vcr/util.py b/tools/vcrpy/vcr/util.py new file mode 100644 index 000000000000..6fc9b2ad9dfb --- /dev/null +++ b/tools/vcrpy/vcr/util.py @@ -0,0 +1,118 @@ +import types + +try: + from collections.abc import Mapping, MutableMapping +except ImportError: + from collections import Mapping, MutableMapping + + +# Shamelessly stolen from https://github.com/kennethreitz/requests/blob/master/requests/structures.py +class CaseInsensitiveDict(MutableMapping): + """ + A case-insensitive ``dict``-like object. + Implements all methods and operations of + ``collections.abc.MutableMapping`` as well as dict's ``copy``. Also + provides ``lower_items``. + All keys are expected to be strings. The structure remembers the + case of the last key to be set, and ``iter(instance)``, + ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` + will contain case-sensitive keys. However, querying and contains + testing is case insensitive:: + cid = CaseInsensitiveDict() + cid['Accept'] = 'application/json' + cid['aCCEPT'] == 'application/json' # True + list(cid) == ['Accept'] # True + For example, ``headers['content-encoding']`` will return the + value of a ``'Content-Encoding'`` response header, regardless + of how the header name was originally stored. + If the constructor, ``.update``, or equality comparison + operations are given keys that have equal ``.lower()``s, the + behavior is undefined. + """ + + def __init__(self, data=None, **kwargs): + self._store = dict() + if data is None: + data = {} + self.update(data, **kwargs) + + def __setitem__(self, key, value): + # Use the lowercased key for lookups, but store the actual + # key alongside the value. + self._store[key.lower()] = (key, value) + + def __getitem__(self, key): + return self._store[key.lower()][1] + + def __delitem__(self, key): + del self._store[key.lower()] + + def __iter__(self): + return (casedkey for casedkey, mappedvalue in self._store.values()) + + def __len__(self): + return len(self._store) + + def lower_items(self): + """Like iteritems(), but with all lowercase keys.""" + return ((lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items()) + + def __eq__(self, other): + if isinstance(other, Mapping): + other = CaseInsensitiveDict(other) + else: + return NotImplemented + # Compare insensitively + return dict(self.lower_items()) == dict(other.lower_items()) + + # Copy is required + def copy(self): + return CaseInsensitiveDict(self._store.values()) + + def __repr__(self): + return str(dict(self.items())) + + +def partition_dict(predicate, dictionary): + true_dict = {} + false_dict = {} + for key, value in dictionary.items(): + this_dict = true_dict if predicate(key, value) else false_dict + this_dict[key] = value + return true_dict, false_dict + + +def compose(*functions): + def composed(incoming): + res = incoming + for function in reversed(functions): + if function: + res = function(res) + return res + + return composed + + +def read_body(request): + if hasattr(request.body, "read"): + return request.body.read() + return request.body + + +def auto_decorate(decorator, predicate=lambda name, value: isinstance(value, types.FunctionType)): + def maybe_decorate(attribute, value): + if predicate(attribute, value): + value = decorator(value) + return value + + class DecorateAll(type): + def __setattr__(cls, attribute, value): + return super(DecorateAll, cls).__setattr__(attribute, maybe_decorate(attribute, value)) + + def __new__(cls, name, bases, attributes_dict): + new_attributes_dict = { + attribute: maybe_decorate(attribute, value) for attribute, value in attributes_dict.items() + } + return super(DecorateAll, cls).__new__(cls, name, bases, new_attributes_dict) + + return DecorateAll From 172407c2d470e0b8076fd63ec16967d8ad0ed78e Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Wed, 7 Jul 2021 09:31:33 -0700 Subject: [PATCH 02/10] add mock after recording response --- tools/vcrpy/vcr/stubs/aiohttp_stubs/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/vcrpy/vcr/stubs/aiohttp_stubs/__init__.py b/tools/vcrpy/vcr/stubs/aiohttp_stubs/__init__.py index 2301334a479c..c8df7a21bb4e 100644 --- a/tools/vcrpy/vcr/stubs/aiohttp_stubs/__init__.py +++ b/tools/vcrpy/vcr/stubs/aiohttp_stubs/__init__.py @@ -116,7 +116,11 @@ async def record_response(cassette, vcr_request, response): """Record a VCR request-response chain to the cassette.""" try: - body = {"string": (await response.read())} + data = await response.read() + body = {"string": (data)} + response.content = MockStream() + response.content.feed_data(data) + response.content.feed_eof() # aiohttp raises a ClientConnectionError on reads when # there is no body. We can use this to know to not write one. except ClientConnectionError: From f29ceb42ff0995687848846d13c23a951a5f0511 Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Wed, 7 Jul 2021 09:33:19 -0700 Subject: [PATCH 03/10] update version number --- tools/vcrpy/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/vcrpy/setup.py b/tools/vcrpy/setup.py index 9f21a8bd0ce4..67ba3899b550 100644 --- a/tools/vcrpy/setup.py +++ b/tools/vcrpy/setup.py @@ -38,7 +38,7 @@ def run_tests(self): setup( name="vcrpy", - version="3.0.0", + version="3.0.1", description=("Automatically mock your HTTP interactions to simplify and " "speed up testing"), long_description=long_description, author="Kevin McCarthy", From 0580c9da248f7ee7b70d7cf0dbc1061b2eb34409 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 7 Jul 2021 11:23:20 -0700 Subject: [PATCH 04/10] use tool version of vcrpy --- eng/ci_tools.txt | 2 +- eng/test_tools.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/ci_tools.txt b/eng/ci_tools.txt index 88a7474e039f..c0b0e74818cc 100644 --- a/eng/ci_tools.txt +++ b/eng/ci_tools.txt @@ -27,7 +27,7 @@ pyOpenSSL==19.1.0 json-delta==2.0 ConfigArgParse==1.2.3 six==1.14.0 -vcrpy==3.0.0 +./tools/vcrpy pyyaml==5.3.1 pytest==5.4.2; python_version >= '3.5' pytest==4.6.9; python_version == '2.7' diff --git a/eng/test_tools.txt b/eng/test_tools.txt index 65c0bfdba6b0..6e824557b808 100644 --- a/eng/test_tools.txt +++ b/eng/test_tools.txt @@ -18,7 +18,7 @@ pyOpenSSL==19.1.0 json-delta==2.0 ConfigArgParse==1.2.3 six==1.14.0 -vcrpy==3.0.0 +./tools/vcrpy pyyaml==5.3.1 packaging==20.4 wheel==0.34.2 From cf7bf45a69815d05a611947641e83f9a44cf1a2f Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Wed, 7 Jul 2021 11:32:05 -0700 Subject: [PATCH 05/10] update --- eng/ci_tools.txt | 2 +- eng/test_tools.txt | 2 +- tools/azure-devtools/setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/ci_tools.txt b/eng/ci_tools.txt index 88a7474e039f..2575b9324fed 100644 --- a/eng/ci_tools.txt +++ b/eng/ci_tools.txt @@ -27,7 +27,7 @@ pyOpenSSL==19.1.0 json-delta==2.0 ConfigArgParse==1.2.3 six==1.14.0 -vcrpy==3.0.0 +vcrpy~=3.0.0 pyyaml==5.3.1 pytest==5.4.2; python_version >= '3.5' pytest==4.6.9; python_version == '2.7' diff --git a/eng/test_tools.txt b/eng/test_tools.txt index 65c0bfdba6b0..eeedaca4f8eb 100644 --- a/eng/test_tools.txt +++ b/eng/test_tools.txt @@ -18,7 +18,7 @@ pyOpenSSL==19.1.0 json-delta==2.0 ConfigArgParse==1.2.3 six==1.14.0 -vcrpy==3.0.0 +vcrpy~=3.0.0 pyyaml==5.3.1 packaging==20.4 wheel==0.34.2 diff --git a/tools/azure-devtools/setup.py b/tools/azure-devtools/setup.py index f4188e8ead57..a39a9b2d6f6d 100644 --- a/tools/azure-devtools/setup.py +++ b/tools/azure-devtools/setup.py @@ -26,7 +26,7 @@ ] -DEPENDENCIES = ["ConfigArgParse>=0.12.0", "six>=1.10.0", "vcrpy==3.0.0"] +DEPENDENCIES = ["ConfigArgParse>=0.12.0", "six>=1.10.0", "vcrpy~=3.0.0"] with io.open("README.rst", "r", encoding="utf-8") as f: README = f.read() From 4541897b757a2ca1d9f946d2cefb1dff16090507 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 7 Jul 2021 12:28:32 -0700 Subject: [PATCH 06/10] patch azure-devtools installer to install patched vcrpy. remove vcrpy from pinned test_tools.txt and ci_tools.txt --- eng/ci_tools.txt | 1 - eng/test_tools.txt | 1 - tools/azure-devtools/setup.py | 10 +++++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/eng/ci_tools.txt b/eng/ci_tools.txt index c0b0e74818cc..1bc5a20fe5ec 100644 --- a/eng/ci_tools.txt +++ b/eng/ci_tools.txt @@ -27,7 +27,6 @@ pyOpenSSL==19.1.0 json-delta==2.0 ConfigArgParse==1.2.3 six==1.14.0 -./tools/vcrpy pyyaml==5.3.1 pytest==5.4.2; python_version >= '3.5' pytest==4.6.9; python_version == '2.7' diff --git a/eng/test_tools.txt b/eng/test_tools.txt index 6e824557b808..ba4de2b1efe8 100644 --- a/eng/test_tools.txt +++ b/eng/test_tools.txt @@ -18,7 +18,6 @@ pyOpenSSL==19.1.0 json-delta==2.0 ConfigArgParse==1.2.3 six==1.14.0 -./tools/vcrpy pyyaml==5.3.1 packaging==20.4 wheel==0.34.2 diff --git a/tools/azure-devtools/setup.py b/tools/azure-devtools/setup.py index a39a9b2d6f6d..6525dbd2c2b4 100644 --- a/tools/azure-devtools/setup.py +++ b/tools/azure-devtools/setup.py @@ -7,11 +7,16 @@ import io from setuptools import setup +import subprocess +import os +import sys +import pdb +patched_vcr = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'vcrpy')) +subprocess.check_call([sys.executable, '-m', 'pip', 'install', patched_vcr]) VERSION = "1.2.1" - CLASSIFIERS = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", @@ -25,8 +30,7 @@ "License :: OSI Approved :: MIT License", ] - -DEPENDENCIES = ["ConfigArgParse>=0.12.0", "six>=1.10.0", "vcrpy~=3.0.0"] +DEPENDENCIES = ["ConfigArgParse>=0.12.0", "six>=1.10.0"] with io.open("README.rst", "r", encoding="utf-8") as f: README = f.read() From 04f3e5ec1a94c160a78ccd880689ac90d0f6b729 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 7 Jul 2021 12:45:10 -0700 Subject: [PATCH 07/10] ensure that ci_tools installs our local builds without erroring on subprocessed vcrpy install --- eng/ci_tools.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/ci_tools.txt b/eng/ci_tools.txt index 1bc5a20fe5ec..4ca3a09cceb6 100644 --- a/eng/ci_tools.txt +++ b/eng/ci_tools.txt @@ -33,5 +33,5 @@ pytest==4.6.9; python_version == '2.7' pytest-cov==2.8.1 # local dev packages -./tools/azure-devtools -./tools/azure-sdk-tools +-e ./tools/azure-devtools +-e ./tools/azure-sdk-tools From be1c07813c3559b83ad1f9eff7ce2ba6060fb54e Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 7 Jul 2021 13:32:28 -0700 Subject: [PATCH 08/10] remove subprocess install. doesn't work without -e which we CANT do in parallelized CI. adding custom req to storage dev reqs --- eng/ci_tools.txt | 4 ++-- sdk/storage/azure-mgmt-storage/dev_requirements.txt | 2 +- sdk/storage/azure-mgmt-storagesync/dev_requirements.txt | 1 + .../azure-storage-blob-changefeed/dev_requirements.txt | 3 ++- sdk/storage/azure-storage-blob/dev_requirements.txt | 1 + .../azure-storage-file-datalake/dev_requirements.txt | 3 ++- .../azure-storage-file-share/dev_requirements.txt | 3 ++- sdk/storage/azure-storage-queue/dev_requirements.txt | 1 + tools/azure-devtools/setup.py | 9 +-------- 9 files changed, 13 insertions(+), 14 deletions(-) diff --git a/eng/ci_tools.txt b/eng/ci_tools.txt index 4ca3a09cceb6..1bc5a20fe5ec 100644 --- a/eng/ci_tools.txt +++ b/eng/ci_tools.txt @@ -33,5 +33,5 @@ pytest==4.6.9; python_version == '2.7' pytest-cov==2.8.1 # local dev packages --e ./tools/azure-devtools --e ./tools/azure-sdk-tools +./tools/azure-devtools +./tools/azure-sdk-tools diff --git a/sdk/storage/azure-mgmt-storage/dev_requirements.txt b/sdk/storage/azure-mgmt-storage/dev_requirements.txt index 59df0e40f18f..7ecca8037d66 100644 --- a/sdk/storage/azure-mgmt-storage/dev_requirements.txt +++ b/sdk/storage/azure-mgmt-storage/dev_requirements.txt @@ -1,4 +1,4 @@ +-e ../../../tools/vcrpy -e ../../../tools/azure-sdk-tools aiohttp>=3.0; python_version >= '3.5' - -e ../../../tools/azure-devtools \ No newline at end of file diff --git a/sdk/storage/azure-mgmt-storagesync/dev_requirements.txt b/sdk/storage/azure-mgmt-storagesync/dev_requirements.txt index 1a1c8d8fc379..3f8e826fb6e1 100644 --- a/sdk/storage/azure-mgmt-storagesync/dev_requirements.txt +++ b/sdk/storage/azure-mgmt-storagesync/dev_requirements.txt @@ -1,2 +1,3 @@ +-e ../../../tools/vcrpy -e ../../../tools/azure-sdk-tools -e ../../../tools/azure-devtools \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob-changefeed/dev_requirements.txt b/sdk/storage/azure-storage-blob-changefeed/dev_requirements.txt index e18a6e446287..2589e88d80f1 100644 --- a/sdk/storage/azure-storage-blob-changefeed/dev_requirements.txt +++ b/sdk/storage/azure-storage-blob-changefeed/dev_requirements.txt @@ -1,5 +1,6 @@ +-e ../../../tools/vcrpy -e ../../../tools/azure-devtools -e ../../../tools/azure-sdk-tools ../azure-storage-blob -e ../../identity/azure-identity -aiohttp>=3.0; python_version >= '3.5' \ No newline at end of file +aiohttp>=3.0; python_version >= '3.5' diff --git a/sdk/storage/azure-storage-blob/dev_requirements.txt b/sdk/storage/azure-storage-blob/dev_requirements.txt index 9938821516f1..bcfe678ce19e 100644 --- a/sdk/storage/azure-storage-blob/dev_requirements.txt +++ b/sdk/storage/azure-storage-blob/dev_requirements.txt @@ -1,3 +1,4 @@ +-e ../../../tools/vcrpy -e ../../../tools/azure-devtools -e ../../../tools/azure-sdk-tools ../../core/azure-core diff --git a/sdk/storage/azure-storage-file-datalake/dev_requirements.txt b/sdk/storage/azure-storage-file-datalake/dev_requirements.txt index 917f034b18e9..fb7d9ba12e0e 100644 --- a/sdk/storage/azure-storage-file-datalake/dev_requirements.txt +++ b/sdk/storage/azure-storage-file-datalake/dev_requirements.txt @@ -1,7 +1,8 @@ +-e ../../../tools/vcrpy -e ../../../tools/azure-devtools -e ../../../tools/azure-sdk-tools ../../core/azure-core ../azure-storage-blob -e ../../identity/azure-identity aiohttp>=3.0; python_version >= '3.5' -adal \ No newline at end of file +adal diff --git a/sdk/storage/azure-storage-file-share/dev_requirements.txt b/sdk/storage/azure-storage-file-share/dev_requirements.txt index 024bd425a0e2..bcfe678ce19e 100644 --- a/sdk/storage/azure-storage-file-share/dev_requirements.txt +++ b/sdk/storage/azure-storage-file-share/dev_requirements.txt @@ -1,5 +1,6 @@ +-e ../../../tools/vcrpy -e ../../../tools/azure-devtools -e ../../../tools/azure-sdk-tools ../../core/azure-core -e ../../identity/azure-identity -aiohttp>=3.0; python_version >= '3.5' \ No newline at end of file +aiohttp>=3.0; python_version >= '3.5' diff --git a/sdk/storage/azure-storage-queue/dev_requirements.txt b/sdk/storage/azure-storage-queue/dev_requirements.txt index 9938821516f1..bcfe678ce19e 100644 --- a/sdk/storage/azure-storage-queue/dev_requirements.txt +++ b/sdk/storage/azure-storage-queue/dev_requirements.txt @@ -1,3 +1,4 @@ +-e ../../../tools/vcrpy -e ../../../tools/azure-devtools -e ../../../tools/azure-sdk-tools ../../core/azure-core diff --git a/tools/azure-devtools/setup.py b/tools/azure-devtools/setup.py index 6525dbd2c2b4..7cbd7512d7dd 100644 --- a/tools/azure-devtools/setup.py +++ b/tools/azure-devtools/setup.py @@ -7,13 +7,6 @@ import io from setuptools import setup -import subprocess -import os -import sys -import pdb - -patched_vcr = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'vcrpy')) -subprocess.check_call([sys.executable, '-m', 'pip', 'install', patched_vcr]) VERSION = "1.2.1" @@ -30,7 +23,7 @@ "License :: OSI Approved :: MIT License", ] -DEPENDENCIES = ["ConfigArgParse>=0.12.0", "six>=1.10.0"] +DEPENDENCIES = ["ConfigArgParse>=0.12.0", "six>=1.10.0", "vcrpy~=3.0.0"] with io.open("README.rst", "r", encoding="utf-8") as f: README = f.read() From 078ef3896c93a02182b28411762d3edfefeaddda Mon Sep 17 00:00:00 2001 From: Azure SDK Bot Date: Wed, 7 Jul 2021 21:03:15 +0000 Subject: [PATCH 09/10] Packaging update of azure-mgmt-storage --- sdk/storage/azure-mgmt-storage/MANIFEST.in | 1 - sdk/storage/azure-mgmt-storage/README.md | 5 +++-- sdk/storage/azure-mgmt-storage/azure/__init__.py | 2 +- sdk/storage/azure-mgmt-storage/azure/mgmt/__init__.py | 2 +- sdk/storage/azure-mgmt-storage/setup.py | 1 + 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sdk/storage/azure-mgmt-storage/MANIFEST.in b/sdk/storage/azure-mgmt-storage/MANIFEST.in index 3a9b6517412b..a3cb07df8765 100644 --- a/sdk/storage/azure-mgmt-storage/MANIFEST.in +++ b/sdk/storage/azure-mgmt-storage/MANIFEST.in @@ -1,4 +1,3 @@ -include _meta.json recursive-include tests *.py *.yaml include *.md include azure/__init__.py diff --git a/sdk/storage/azure-mgmt-storage/README.md b/sdk/storage/azure-mgmt-storage/README.md index f5c491e31761..4322f2e24f9f 100644 --- a/sdk/storage/azure-mgmt-storage/README.md +++ b/sdk/storage/azure-mgmt-storage/README.md @@ -12,15 +12,16 @@ To learn how to use this package, see the [quickstart guide](https://aka.ms/azsd -For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure/) +For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure/storage?view=azure-python-preview) Code samples for this package can be found at [Storage Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. Additional code samples for different Azure services are available at [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) + # Provide Feedback If you encounter any bugs or have suggestions, please file an issue in the [Issues](https://github.com/Azure/azure-sdk-for-python/issues) -section of the project. +section of the project. ![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2Fazure-mgmt-storage%2FREADME.png) diff --git a/sdk/storage/azure-mgmt-storage/azure/__init__.py b/sdk/storage/azure-mgmt-storage/azure/__init__.py index 0260537a02bb..8db66d3d0f0f 100644 --- a/sdk/storage/azure-mgmt-storage/azure/__init__.py +++ b/sdk/storage/azure-mgmt-storage/azure/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/sdk/storage/azure-mgmt-storage/azure/mgmt/__init__.py b/sdk/storage/azure-mgmt-storage/azure/mgmt/__init__.py index 0260537a02bb..8db66d3d0f0f 100644 --- a/sdk/storage/azure-mgmt-storage/azure/mgmt/__init__.py +++ b/sdk/storage/azure-mgmt-storage/azure/mgmt/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/sdk/storage/azure-mgmt-storage/setup.py b/sdk/storage/azure-mgmt-storage/setup.py index 86e471425699..c4a4909c9edf 100644 --- a/sdk/storage/azure-mgmt-storage/setup.py +++ b/sdk/storage/azure-mgmt-storage/setup.py @@ -70,6 +70,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: MIT License', ], zip_safe=False, From febf0f03dd9d7ad63235b37d83d9c9f94c15d2e1 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot Date: Wed, 7 Jul 2021 21:03:16 +0000 Subject: [PATCH 10/10] Packaging update of azure-mgmt-storagesync --- .../azure-mgmt-storagesync/MANIFEST.in | 1 - sdk/storage/azure-mgmt-storagesync/README.md | 30 +++++++------------ .../azure-mgmt-storagesync/azure/__init__.py | 2 +- .../azure/mgmt/__init__.py | 2 +- sdk/storage/azure-mgmt-storagesync/setup.py | 7 +++-- 5 files changed, 18 insertions(+), 24 deletions(-) diff --git a/sdk/storage/azure-mgmt-storagesync/MANIFEST.in b/sdk/storage/azure-mgmt-storagesync/MANIFEST.in index 3a9b6517412b..a3cb07df8765 100644 --- a/sdk/storage/azure-mgmt-storagesync/MANIFEST.in +++ b/sdk/storage/azure-mgmt-storagesync/MANIFEST.in @@ -1,4 +1,3 @@ -include _meta.json recursive-include tests *.py *.yaml include *.md include azure/__init__.py diff --git a/sdk/storage/azure-mgmt-storagesync/README.md b/sdk/storage/azure-mgmt-storagesync/README.md index 945dfc1a4da4..99dd52105dd1 100644 --- a/sdk/storage/azure-mgmt-storagesync/README.md +++ b/sdk/storage/azure-mgmt-storagesync/README.md @@ -1,35 +1,27 @@ -## Microsoft Azure SDK for Python +# Microsoft Azure SDK for Python This is the Microsoft Azure Storage Sync Client Library. - -Azure Resource Manager (ARM) is the next generation of management APIs -that replace the old Azure Service Management (ASM). - This package has been tested with Python 2.7, 3.5, 3.6, 3.7 and 3.8. +For a more complete view of Azure libraries, see the [azure sdk python release](https://aka.ms/azsdk/python/all). -For the older Azure Service Management (ASM) libraries, see -[azure-servicemanagement-legacy](https://pypi.python.org/pypi/azure-servicemanagement-legacy) -library. - -For a more complete set of Azure libraries, see the -[azure sdk python release](https://aka.ms/azsdk/python/all). -## Usage +# Usage To learn how to use this package, see the [quickstart guide](https://aka.ms/azsdk/python/mgmt) - + For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure/) -Code samples for this package can be found at [Storage Sync Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. +Code samples for this package can be found at [Storage Sync](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. Additional code samples for different Azure services are available at [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) -## Provide Feedback +# Provide Feedback + +If you encounter any bugs or have suggestions, please file an issue in the +[Issues](https://github.com/Azure/azure-sdk-for-python/issues) +section of the project. -If you encounter any bugs or have suggestions, please file an issue in -the [Issues](https://github.com/Azure/azure-sdk-for-python/issues) -section of the project. -![image](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2Fazure-mgmt-storagesync%2FREADME.png) +![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2Fazure-mgmt-storagesync%2FREADME.png) diff --git a/sdk/storage/azure-mgmt-storagesync/azure/__init__.py b/sdk/storage/azure-mgmt-storagesync/azure/__init__.py index 0260537a02bb..8db66d3d0f0f 100644 --- a/sdk/storage/azure-mgmt-storagesync/azure/__init__.py +++ b/sdk/storage/azure-mgmt-storagesync/azure/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/sdk/storage/azure-mgmt-storagesync/azure/mgmt/__init__.py b/sdk/storage/azure-mgmt-storagesync/azure/mgmt/__init__.py index 0260537a02bb..8db66d3d0f0f 100644 --- a/sdk/storage/azure-mgmt-storagesync/azure/mgmt/__init__.py +++ b/sdk/storage/azure-mgmt-storagesync/azure/mgmt/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/sdk/storage/azure-mgmt-storagesync/setup.py b/sdk/storage/azure-mgmt-storagesync/setup.py index bc9bf4d4ff8c..4042f659ffad 100644 --- a/sdk/storage/azure-mgmt-storagesync/setup.py +++ b/sdk/storage/azure-mgmt-storagesync/setup.py @@ -36,7 +36,9 @@ pass # Version extraction inspired from 'requests' -with open(os.path.join(package_folder_path, '_version.py'), 'r') as fd: +with open(os.path.join(package_folder_path, 'version.py') + if os.path.exists(os.path.join(package_folder_path, 'version.py')) + else os.path.join(package_folder_path, '_version.py'), 'r') as fd: version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) @@ -68,6 +70,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: MIT License', ], zip_safe=False, @@ -79,8 +82,8 @@ ]), install_requires=[ 'msrest>=0.6.21', + 'msrestazure>=0.4.32,<2.0.0', 'azure-common~=1.1', - 'azure-mgmt-core>=1.2.0,<2.0.0', ], extras_require={ ":python_version<'3.0'": ['azure-mgmt-nspkg'],