diff --git a/.gitignore b/.gitignore index 25fa4cf..a17e642 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ lib/*.egg-info .tox .coverage .eggs +.mypy_cache +venv diff --git a/.travis.yml b/.travis.yml index 75d4057..6340aba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,10 @@ matrix: env: TOXENV=flake8 dist: xenial sudo: true + - python: 3.7 + env: TOXENV=mypy + dist: xenial + sudo: true before_install: - git submodule update --init install: diff --git a/Makefile b/Makefile index cce34f9..a0024c8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ -all: test +.PHONY: test flake8 mypy autopep8 +all: autopep8 test mypy flake8 TIMESTAMP=$(shell date +%Y%m%d-%H%M%S) @@ -8,3 +9,12 @@ lib/woothee/dataset.py: woothee/dataset.yaml test: lib/woothee/dataset.py python setup.py test + +flake8: + tox -eflake8 + +mypy: + tox -emypy + +autopep8: + tox -eautopep8 diff --git a/README.rst b/README.rst index e30a4d9..fcb96c4 100644 --- a/README.rst +++ b/README.rst @@ -96,6 +96,11 @@ limitations under the License. History ------- +1.10.0(Apr 12, 2019) +~~~~~~~~~~~~~~~~~~~~ + +* `#15 Support for v1.10.0 `_ + 1.8.0(Jul 5, 2018) ~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/woothee/__init__.py b/lib/woothee/__init__.py index 9882307..077997d 100644 --- a/lib/woothee/__init__.py +++ b/lib/woothee/__init__.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import (division, print_function, absolute_import, unicode_literals) +from typing import Dict from . import dataset from . import browser @@ -10,8 +11,8 @@ from . import appliance from . import misc -VERSION = (1, 8, 0) -__version__ = '1.8.0' +VERSION = (1, 10, 0) +__version__ = '1.10.0' FILLED = { dataset.ATTRIBUTE_NAME: dataset.VALUE_UNKNOWN, @@ -32,7 +33,7 @@ def is_crawler(useragent): def exec_parse(useragent): - result = {} + result = {} # type: Dict[str, str] if not useragent or useragent == '-': return result diff --git a/lib/woothee/__init__.pyi b/lib/woothee/__init__.pyi new file mode 100644 index 0000000..90e5c12 --- /dev/null +++ b/lib/woothee/__init__.pyi @@ -0,0 +1,20 @@ +# Stubs for woothee (Python 3) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Text, AnyStr, Tuple, Dict + +VERSION: Tuple +FILLED: Dict[Text, Text] + +def parse(useragent: AnyStr) -> Dict[AnyStr, AnyStr]: ... +def is_crawler(useragent: AnyStr) -> bool: ... +def exec_parse(useragent: AnyStr) -> Dict[AnyStr, AnyStr]: ... +def try_crawler(useragent: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def try_browser(useragent: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def try_os(useragent: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def try_mobilephone(useragent: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def try_appliance(useragent: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def try_misc(useragent: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def try_rare_cases(useragent: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def fill_result(result: Dict[AnyStr, AnyStr]) -> Dict[AnyStr, AnyStr]: ... diff --git a/lib/woothee/appliance.pyi b/lib/woothee/appliance.pyi new file mode 100644 index 0000000..4d35c26 --- /dev/null +++ b/lib/woothee/appliance.pyi @@ -0,0 +1,9 @@ +# Stubs for woothee.appliance (Python 3) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Dict, AnyStr + +def challenge_playstation(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_nintendo(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_digitaltv(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... diff --git a/lib/woothee/browser.py b/lib/woothee/browser.py index 93f1a3d..108db40 100644 --- a/lib/woothee/browser.py +++ b/lib/woothee/browser.py @@ -49,7 +49,7 @@ def challenge_safari_chrome(ua, result): version = dataset.VALUE_UNKNOWN # Edge - obj = re.search(r'Edge\/([.0-9]+)', ua) + obj = re.search(r'(?:Edge|Edg|EdgiOS|EdgA)\/([.0-9]+)', ua) if obj: version = obj.group(1) util.update_map(result, dataset.get('Edge')) diff --git a/lib/woothee/browser.pyi b/lib/woothee/browser.pyi new file mode 100644 index 0000000..80cce6d --- /dev/null +++ b/lib/woothee/browser.pyi @@ -0,0 +1,14 @@ +# Stubs for woothee.browser (Python 3) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Dict, AnyStr + +def challenge_msie(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_yandexbrowser(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_safari_chrome(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_firefox(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_opera(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_webview(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_sleipnir(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_vivaldi(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... diff --git a/lib/woothee/crawler.pyi b/lib/woothee/crawler.pyi new file mode 100644 index 0000000..d5629c4 --- /dev/null +++ b/lib/woothee/crawler.pyi @@ -0,0 +1,9 @@ +# Stubs for woothee.crawler (Python 3) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Dict, AnyStr + +def challenge_google(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_crawlers(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_maybe_crawler(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... diff --git a/lib/woothee/dataset.py b/lib/woothee/dataset.py index 6f34b34..8a9d6eb 100644 --- a/lib/woothee/dataset.py +++ b/lib/woothee/dataset.py @@ -42,7 +42,7 @@ def _init(): - # GENERATED from dataset.yaml at Wed Jul 4 23:02:52 2018 by hattori + # GENERATED from dataset.yaml at Fri Apr 12 23:12:13 2019 by tell_k obj = {'label': 'MSIE', 'name': 'Internet Explorer', 'type': 'browser'} # NOQA obj['vendor'] = 'Microsoft' DATASET[obj['label']] = obj diff --git a/lib/woothee/dataset.pyi b/lib/woothee/dataset.pyi new file mode 100644 index 0000000..af5a44c --- /dev/null +++ b/lib/woothee/dataset.pyi @@ -0,0 +1,35 @@ +# Stubs for woothee.dataset (Python 3) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Text, List, Dict + +KEY_LABEL: Text +KEY_NAME: Text +KEY_TYPE: Text +KEY_CATEGORY: Text +KEY_OS: Text +KEY_OS_VERSION: Text +KEY_VENDOR: Text +KEY_VERSION: Text +TYPE_BROWSER: Text +TYPE_OS: Text +TYPE_FULL: Text +CATEGORY_PC: Text +CATEGORY_SMARTPHONE: Text +CATEGORY_MOBILEPHONE: Text +CATEGORY_CRAWLER: Text +CATEGORY_APPLIANCE: Text +CATEGORY_MISC: Text +ATTRIBUTE_NAME: Text +ATTRIBUTE_CATEGORY: Text +ATTRIBUTE_OS: Text +ATTRIBUTE_OS_VERSION: Text +ATTRIBUTE_VENDOR: Text +ATTRIBUTE_VERSION: Text +VALUE_UNKNOWN: Text +CATEGORY_LIST: List[Text] +ATTRIBUTE_LIST: List[Text] +DATASET: Dict[Text, Text] + +def get(label: Text) -> Dict[Text, Text]: ... diff --git a/lib/woothee/misc.pyi b/lib/woothee/misc.pyi new file mode 100644 index 0000000..7cf9d76 --- /dev/null +++ b/lib/woothee/misc.pyi @@ -0,0 +1,10 @@ +# Stubs for woothee.misc (Python 3) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Dict, AnyStr + +def challenge_desktoptools(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_smartphone_patterns(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_http_library(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_maybe_rss_reader(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... diff --git a/lib/woothee/mobilephone.pyi b/lib/woothee/mobilephone.pyi new file mode 100644 index 0000000..bbdfc3c --- /dev/null +++ b/lib/woothee/mobilephone.pyi @@ -0,0 +1,11 @@ +# Stubs for woothee.mobilephone (Python 3) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Dict, AnyStr + +def challenge_docomo(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_au(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_softbank(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_willcom(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_misc(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... diff --git a/lib/woothee/os.py b/lib/woothee/os.py index eb903d0..0e382e0 100644 --- a/lib/woothee/os.py +++ b/lib/woothee/os.py @@ -105,7 +105,7 @@ def challenge_linux(ua, result): os_version = None if 'Android' in ua: data = dataset.get('Android') - regex = re.compile(r"Android[- ](\d+\.\d+(?:\.\d+)?)") + regex = re.compile(r"Android[- ](\d+(?:\.\d+(?:\.\d+)?)?)") m = regex.search(ua) if m: os_version = m.group(1) diff --git a/lib/woothee/os.pyi b/lib/woothee/os.pyi new file mode 100644 index 0000000..e66a3b3 --- /dev/null +++ b/lib/woothee/os.pyi @@ -0,0 +1,13 @@ +# Stubs for woothee.os (Python 3) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Dict, AnyStr + +def challenge_windows(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_osx(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_linux(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_smartphone(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_mobilephone(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_appliance(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... +def challenge_misc(ua: AnyStr, result: Dict[AnyStr, AnyStr]) -> bool: ... diff --git a/lib/woothee/py.typed b/lib/woothee/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/lib/woothee/util.pyi b/lib/woothee/util.pyi new file mode 100644 index 0000000..9bb991c --- /dev/null +++ b/lib/woothee/util.pyi @@ -0,0 +1,11 @@ +# Stubs for woothee.util (Python 3) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Dict, AnyStr + +def update_map(target: Dict[AnyStr, AnyStr], source: Dict[AnyStr, AnyStr]) -> None: ... +def update_category(target: Dict[AnyStr, AnyStr], category: AnyStr) -> None: ... +def update_version(target: Dict[AnyStr, AnyStr], version: AnyStr) -> None: ... +def update_os(target: Dict[AnyStr, AnyStr], os: AnyStr) -> None: ... +def update_os_version(target: Dict[AnyStr, AnyStr], version: AnyStr) -> None: ... diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..95e584c --- /dev/null +++ b/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +python_version = 2.7 +ignore_missing_imports = True +incremental = True +check_untyped_defs = True diff --git a/scripts/dataset_yaml2py.py b/scripts/dataset_yaml2py.py index da93c4f..36a8ee8 100644 --- a/scripts/dataset_yaml2py.py +++ b/scripts/dataset_yaml2py.py @@ -26,7 +26,7 @@ fp = open(dataset_file, 'rb') try: - for datasets in yaml.load_all(fp): + for datasets in yaml.safe_load_all(fp): for dataset in datasets: label = dataset['label'] name = dataset['name'] diff --git a/setup.cfg b/setup.cfg index 1659b70..a91719d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,3 +9,8 @@ strict = 1 [aliases] release = sdist bdist_wheel +test=pytest + +[tool:pytest] +addopts = -vv --cov lib/woothee --cov-report term-missing +python_files = tests/*.py diff --git a/setup.py b/setup.py index ffe1a48..de4f57a 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ import os import re +import sys from setuptools import setup, Command, find_packages @@ -20,18 +21,24 @@ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Python Modules", ] +install_requires = ['six>=1.8.0'] + +if sys.version_info < (3, 5): + install_requires.append('typing') + + class DatasetCommand(Command): description = 'generate dataset.py' @@ -51,21 +58,24 @@ def run(self): sys.path.insert(0, scripts_dir) import dataset_yaml2py # NOQA + setup( name='woothee', version=version, - description='Cross-language UserAgent classifier library, python implementation', # NOQA + description='Cross-language UserAgent classifier library, python implementation', # NOQA author='tell-k', author_email='ffk2005@gmail.com', url='https://github.com/woothee/woothee-python', license='Apache License 2.0', packages=find_packages('lib'), package_dir={'': 'lib'}, + package_data={ + 'woothee': ['py.typed', '*.pyi'], + }, platforms='any', - install_requires=["six>=1.8.0"], - setup_requires=['PyYAML>=3.10', "six>=1.8.0"], - tests_require=['mock'], - test_suite='tests', + install_requires=install_requires, + setup_requires=['PyYAML>=3.10', 'six>=1.8.0', 'pytest-runner'], + tests_require=['pytest', 'pytest-cov', 'pytest-mock'], long_description=long_description, classifiers=classifiers, keywords=['web', 'user-agent', 'parser'], diff --git a/tests/test.py b/tests/test.py index 587b42e..d8662a2 100644 --- a/tests/test.py +++ b/tests/test.py @@ -1,276 +1,293 @@ # -*- coding:utf-8 -*- from __future__ import (division, print_function, absolute_import, unicode_literals) - -import mock import os import sys -import unittest + import yaml +import pytest + +from typing import Dict # NOQA BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(BASE_PATH, 'lib')) +sys.path.insert(0, os.path.join(BASE_PATH, 'lib')) # type: ignore TESTSET_DIR = os.path.join(BASE_PATH, 'woothee', 'testsets') TARGETS = [ + ['appliance.yaml', 'Appliance'], + ['blank.yaml', 'Blank'], ['crawler.yaml', 'Crawler'], ['crawler_google.yaml', 'Crawler/Google'], - ['pc_windows.yaml', 'PC/Windows'], - ['pc_misc.yaml', 'PC/Misc'], - ['mobilephone_docomo.yaml', 'MobilePhone/docomo'], + ['crawler_nonmajor.yaml', 'Crawler/NonMajor'], + ['misc.yaml', 'Misc'], ['mobilephone_au.yaml', 'MobilePhone/au'], + ['mobilephone_docomo.yaml', 'MobilePhone/docomo'], + ['mobilephone_misc.yaml', 'MobilePhone/misc'], ['mobilephone_softbank.yaml', 'MobilePhone/softbank'], ['mobilephone_willcom.yaml', 'MobilePhone/willcom'], - ['mobilephone_misc.yaml', 'MobilePhone/misc'], - ['smartphone_ios.yaml', 'SmartPhone/ios'], + ['pc_lowpriority.yaml', 'PC/LowPriority'], + ['pc_misc.yaml', 'PC/Misc'], + ['pc_windows.yaml', 'PC/Windows'], ['smartphone_android.yaml', 'SmartPhone/android'], + ['smartphone_ios.yaml', 'SmartPhone/ios'], ['smartphone_misc.yaml', 'SmartPhone/misc'], - ['appliance.yaml', 'Appliance'], - ['pc_lowpriority.yaml', 'PC/LowPriority'], - ['misc.yaml', 'Misc'], - ['crawler_nonmajor.yaml', 'Crawler/NonMajor'], - ['blank.yaml', 'Blank'], ] -class WootheeTest(unittest.TestCase): +def gen_test_cases(): + for filename, groupname in TARGETS: + with open(os.path.join(TESTSET_DIR, filename), 'rb') as fp: + for test_cases in yaml.safe_load_all(fp): + for test_case in test_cases: + yield groupname, test_case + + +class TestDataset: def test_contains_constants(self): from woothee import dataset - self.assertEqual(dataset.ATTRIBUTE_NAME, 'name') + assert dataset.ATTRIBUTE_NAME == 'name' - def test_contains_const_list(self): + def test_contains_attribute_list(self): from woothee import dataset - self.assertEqual( - dataset.ATTRIBUTE_LIST, - [dataset.ATTRIBUTE_NAME, dataset.ATTRIBUTE_CATEGORY, - dataset.ATTRIBUTE_OS, dataset.ATTRIBUTE_VENDOR, - dataset.ATTRIBUTE_VERSION, dataset.ATTRIBUTE_OS_VERSION] - ) - self.assertEqual( - dataset.CATEGORY_LIST, - [dataset.CATEGORY_PC, dataset.CATEGORY_SMARTPHONE, - dataset.CATEGORY_MOBILEPHONE, dataset.CATEGORY_CRAWLER, - dataset.CATEGORY_APPLIANCE, dataset.CATEGORY_MISC, - dataset.VALUE_UNKNOWN] - ) - - def test_contains_dataset(self): + assert dataset.ATTRIBUTE_LIST == [ + dataset.ATTRIBUTE_NAME, + dataset.ATTRIBUTE_CATEGORY, + dataset.ATTRIBUTE_OS, + dataset.ATTRIBUTE_VENDOR, + dataset.ATTRIBUTE_VERSION, + dataset.ATTRIBUTE_OS_VERSION + ] + + def test_contains_category_list(self): from woothee import dataset - self.assertEqual(dataset.get('GoogleBot')['name'], 'Googlebot') - def test_should_be_read_from_each_modules_correctly(self): - pass + assert dataset.CATEGORY_LIST == [ + dataset.CATEGORY_PC, + dataset.CATEGORY_SMARTPHONE, + dataset.CATEGORY_MOBILEPHONE, + dataset.CATEGORY_CRAWLER, + dataset.CATEGORY_APPLIANCE, + dataset.CATEGORY_MISC, + dataset.VALUE_UNKNOWN + ] - def test_testsets(self): - import woothee - check_attributes = ('name', 'category', 'os', - 'version', 'vendor', 'os_version') +class TestParse: - def _can_assert(attribute, e): - if attribute in ('name', 'category'): - return True - check_attributes = ('os', 'version', 'vendor', 'os_version') - if attribute in check_attributes and attribute in e: - return True - return False + @pytest.fixture() + def target(self): + from woothee import parse + return parse - for filename, groupname in TARGETS: - with open(os.path.join(TESTSET_DIR, filename), 'rb') as fp: - for es in yaml.load_all(fp): - for e in es: - r = woothee.parse(e['target']) - for attribute in check_attributes: + @pytest.mark.parametrize(('groupname', 'test_case'), gen_test_cases()) + def test_testsets(self, target, groupname, test_case): + ua_string = test_case.pop('target') + expected = test_case - testname = groupname + (' test({0}): {1}'.format( - attribute, e['target'] - )) - if _can_assert(attribute, e): - self.assertEqual( - r[attribute], - e[attribute], - testname - ) + parsed = target(ua_string) - def test_non_provide_testsets(self): - # This test pattern that does not exist in testsets. - # The main purpose is that each logic to pass. - # UserAgent is a dummy that does not exist in the world. - - import woothee + # Check only the attrs exists in the expected(=test_case). + actual = {k: v for k, v in parsed.items() if k in expected} + msg = '{0} test({1})'.format(groupname, ua_string) + assert actual == expected, msg + @pytest.mark.parametrize(('expected', 'ua_string'), [ # 48 line in lib/woothee/appliance.py - self.assertEqual({ - "name": "Nintendo DSi", - "version": "UNKNOWN", - "os": "Nintendo DSi", - "os_version": "UNKNOWN", - "category": "appliance", - "vendor": "Nintendo", - }, woothee.parse("(Nintendo DSi; U; ja)")) - + ( + { + "name": "Nintendo DSi", + "version": "UNKNOWN", + "os": "Nintendo DSi", + "os_version": "UNKNOWN", + "category": "appliance", + "vendor": "Nintendo", + }, + "(Nintendo DSi; U; ja)" + ), # 50 line in lib/woothee/appliance.py - self.assertEqual({ - "name": "Nintendo Wii", - "version": "UNKNOWN", - "os": "Nintendo Wii", - "os_version": "UNKNOWN", - "category": "appliance", - "vendor": "Nintendo", - }, woothee.parse("(Nintendo Wii; U; ; 3642; ja)")) - + ( + { + "name": "Nintendo Wii", + "version": "UNKNOWN", + "os": "Nintendo Wii", + "os_version": "UNKNOWN", + "category": "appliance", + "vendor": "Nintendo", + }, + "(Nintendo Wii; U; ; 3642; ja)" + ), # 26 line lib/woothee/browser.py - self.assertEqual({ - "name": "Internet Explorer", - "version": "11.0", - "os": "Windows Phone OS", - "os_version": "8.1", - "category": "smartphone", - "vendor": "Microsoft", - }, woothee.parse( - "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0;" - " Touch; IEMobile/11.0; NOKIA; Lumia 930) like Gecko" - )) - + ( + { + "name": "Internet Explorer", + "version": "11.0", + "os": "Windows Phone OS", + "os_version": "8.1", + "category": "smartphone", + "vendor": "Microsoft", + }, + ( + "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0;" + " Touch; IEMobile/11.0; NOKIA; Lumia 930) like Gecko" + ) + ), # 159 line lib/woothee/crawler.py - self.assertEqual({ - "name": "UNKNOWN", - "version": "UNKNOWN", - "os": "UNKNOWN", - "os_version": "UNKNOWN", - "category": "UNKNOWN", - "vendor": "UNKNOWN", - }, woothee.parse("Data-Hotel-Cat/1.1")) - + ( + { + "name": "UNKNOWN", + "version": "UNKNOWN", + "os": "UNKNOWN", + "os_version": "UNKNOWN", + "category": "UNKNOWN", + "vendor": "UNKNOWN", + }, + "Data-Hotel-Cat/1.1" + ), # 74-75 line lib/woothee/mobilephone.py - self.assertEqual({ - "name": "SymbianOS", - "version": "UNKNOWN", - "os": "SymbianOS", - "os_version": "UNKNOWN", - "category": "mobilephone", - "vendor": "UNKNOWN", - }, woothee.parse("SymbianOS/9.2;")) - + ( + { + "name": "SymbianOS", + "version": "UNKNOWN", + "os": "SymbianOS", + "os_version": "UNKNOWN", + "category": "mobilephone", + "vendor": "UNKNOWN", + }, + "SymbianOS/9.2;" + ), # 78-80 line lib/woothee/mobilephone.py - self.assertEqual({ - "name": "Mobile Transcoder", - "version": "Hatena", - "os": "Mobile Transcoder", - "os_version": "UNKNOWN", - "category": "mobilephone", - "vendor": "UNKNOWN", - }, woothee.parse( - "(compatible; Hatena-Mobile-Gateway/1.2;" - " +http://mgw.hatena.ne.jp/help)" - )) - + ( + { + "name": "Mobile Transcoder", + "version": "Hatena", + "os": "Mobile Transcoder", + "os_version": "UNKNOWN", + "category": "mobilephone", + "vendor": "UNKNOWN", + }, + ( + "(compatible; Hatena-Mobile-Gateway/1.2;" + " +http://mgw.hatena.ne.jp/help)" + ) + ), # 25-27 line lib/woothee/os.py - self.assertEqual({ - "name": "UNKNOWN", - "version": "UNKNOWN", - "os": "Windows UNKNOWN Ver", - "os_version": "UNKNOWN", - "category": "pc", - "vendor": "UNKNOWN", - }, woothee.parse( + ( + { + "name": "UNKNOWN", + "version": "UNKNOWN", + "os": "Windows UNKNOWN Ver", + "os_version": "UNKNOWN", + "category": "pc", + "vendor": "UNKNOWN", + }, "Mozilla/5.0 (Windows ; rv:8.0) Gecko/20111105 Thunderbird/8.0" - )) - + ), # 49 line lib/woothee/os.py - self.assertEqual({ - "name": "Internet Explorer", - "version": "UNKNOWN", - "os": "Windows NT 4.0", - "os_version": "NT 4.0", - "category": "pc", - "vendor": "Microsoft", - }, woothee.parse( + ( + { + "name": "Internet Explorer", + "version": "UNKNOWN", + "os": "Windows NT 4.0", + "os_version": "NT 4.0", + "category": "pc", + "vendor": "Microsoft", + }, "Mozilla/4.0 (compatible; MSIE 6.0b; Windows NT 4.0)" - )) - + ), # 51 line lib/woothee/os.py - self.assertEqual({ - "name": "Internet Explorer", - "version": "6.0", - "os": "Windows 98", - "os_version": "98", - "category": "pc", - "vendor": "Microsoft", - }, woothee.parse( + ( + { + "name": "Internet Explorer", + "version": "6.0", + "os": "Windows 98", + "os_version": "98", + "category": "pc", + "vendor": "Microsoft", + }, "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)" - )) - + ), # 53 line lib/woothee/os.py - self.assertEqual({ - "name": "Internet Explorer", - "version": "5.50", - "os": "Windows 95", - "os_version": "95", - "category": "pc", - "vendor": "Microsoft", - }, woothee.parse( + ( + { + "name": "Internet Explorer", + "version": "5.50", + "os": "Windows 95", + "os_version": "95", + "category": "pc", + "vendor": "Microsoft", + }, "Mozilla/4.0 (compatible; MSIE 5.50; Windows 95; SiteKiosk 4.8)" - )) - + ), # 121 line lib/woothee/os.py - self.assertEqual({ - "name": "UNKNOWN", - "version": "UNKNOWN", - "os": "iPad", - "os_version": "UNKNOWN", - "category": "smartphone", - "vendor": "UNKNOWN", - }, woothee.parse("Mozilla/5.0 (iPad; ")) - + ( + { + "name": "UNKNOWN", + "version": "UNKNOWN", + "os": "iPad", + "os_version": "UNKNOWN", + "category": "smartphone", + "vendor": "UNKNOWN", + }, + "Mozilla/5.0 (iPad; " + ), # 123 line lib/woothee/os.py - self.assertEqual({ - "name": "UNKNOWN", - "version": "UNKNOWN", - "os": "iPod", - "os_version": "UNKNOWN", - "category": "smartphone", - "vendor": "UNKNOWN", - }, woothee.parse("Mozilla/5.0 (iPod; ")) - + ( + { + "name": "UNKNOWN", + "version": "UNKNOWN", + "os": "iPod", + "os_version": "UNKNOWN", + "category": "smartphone", + "vendor": "UNKNOWN", + }, + "Mozilla/5.0 (iPod; " + ), # 183-185 line lib/woothee/os.py - self.assertEqual({ - "name": "Mobile Transcoder", - "version": "Naver", - "os": "Mobile Transcoder", - "os_version": "UNKNOWN", - "category": "mobilephone", - "vendor": "UNKNOWN", - }, woothee.parse("Naver Transcoder")) + ( + { + "name": "Mobile Transcoder", + "version": "Naver", + "os": "Mobile Transcoder", + "os_version": "UNKNOWN", + "category": "mobilephone", + "vendor": "UNKNOWN", + }, + "Naver Transcoder" + ) + ]) + def test_non_provide_testsets(self, target, expected, ua_string): + # This test pattern that does not exist in testsets. + # The main purpose is that each logic to pass. + # UserAgent is a dummy that does not exist in the world. + assert expected == target(ua_string) -class TestIsCrawler(unittest.TestCase): +class TestIsCrawler: - def _getTarget(self): + @pytest.fixture + def target(self): from woothee import is_crawler return is_crawler - def _callFUT(self, *args, **kwargs): - return self._getTarget()(*args, **kwargs) - - def test_false(self): - self.assertFalse(self._callFUT("")) - self.assertFalse(self._callFUT("-")) - self.assertFalse(self._callFUT(None)) - self.assertFalse(self._callFUT( - "Mozilla/5.0 (Windows NT 6.3; " - "Trident/7.0; rv:11.0) like Gecko" - )) + @pytest.mark.parametrize('ua_string', [ + "", + "-", + None, + "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko" + ]) + def test_false(self, target, ua_string): + assert not target(ua_string) - def test_true(self): - self.assertTrue(self._callFUT( - "Mozilla/5.0 (compatible; Yahoo! Slurp;" - " http://help.yahoo.com/help/us/ysearch/slurp)" - )) + @pytest.mark.parametrize('ua_string', [ + ("Mozilla/5.0 (compatible; Yahoo! Slurp;" + " http://help.yahoo.com/help/us/ysearch/slurp)"), + ]) + def test_true(self, target, ua_string): + assert target(ua_string) -class TestTryRareCases(unittest.TestCase): +class TestTryRareCases: """ challenge_smartphone_patterns in try_rare_cases is never return True. Because, "CFNetwork" is caught by the challenge_smartphone in try_os. Therefore, I have prepared the individual a test case @@ -278,20 +295,25 @@ class TestTryRareCases(unittest.TestCase): Not need this function(challenge_smartphone_patterns) just maybe. """ - def _getTarget(self): + @pytest.fixture() + def target(self): from woothee import try_rare_cases return try_rare_cases - def _callFUT(self, *args, **kwargs): - return self._getTarget()(*args, **kwargs) - - @mock.patch("woothee.browser.challenge_sleipnir") - def test_challenge_smartphone_patterns(self, mock_sleipnir): - result = {} - self.assertTrue(self._callFUT("CFNetwork/", result)) - self.assertEqual({'category': 'smartphone', 'os': 'iOS'}, result) - self.assertFalse(mock_sleipnir.called) - - -if __name__ == '__main__': - unittest.main() + def test_challenge_smartphone_patterns(self, target, mocker): + m = mocker.patch("woothee.browser.challenge_sleipnir") + result = {} # type: Dict[str, str] + + ret = target("CFNetwork/", result) + + expected = { + 'return_value': True, + 'result': {'category': 'smartphone', 'os': 'iOS'}, + 'called_check': False, + } + actual = { + 'return_value': ret, + 'result': result, + 'called_check': m.called, + } + assert expected == actual diff --git a/tox.ini b/tox.ini index b55ac0a..c32f486 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,23 @@ [tox] -envlist=py26,py27,py32,py33,py34,py35,py36,py37,pypy,flake8 +envlist=py27,py34,py35,py36,py37,pypy,flake8,mypy,autopep8 [testenv] -deps = coverage commands= - coverage erase - coverage run --source lib/woothee setup.py test - coverage report -m - - -# latest coverage package dropped Python3.2 support. -[testenv:py32] -deps = coverage==3.7.1 + python setup.py test [testenv:flake8] deps = flake8 commands= flake8 lib tests scripts + +[testenv:mypy] +basepython = python3 +deps = + mypy +commands = mypy lib tests + +[testenv:autopep8] +basepython = python3 +deps = + autopep8 +commands = autopep8 --in-place --aggressive --aggressive --recursive lib/ tests/ diff --git a/woothee b/woothee index 6f94012..1d21d7c 160000 --- a/woothee +++ b/woothee @@ -1 +1 @@ -Subproject commit 6f94012965dc21b4ba3b33c4c4113a00e4e2dffb +Subproject commit 1d21d7cbfd57b568e964cdd2ef98f9e046512628