Skip to content

Use pkg_resources to parse requirements #6158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bsiegel merged 1 commit into from
Jun 28, 2019
Merged

Use pkg_resources to parse requirements #6158

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 24 additions & 22 deletions scripts/analyze_deps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import glob
import io
import os
from pkg_resources import Requirement

@scbedd scbedd Jun 28, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh this is awesome. setuptools is legit

import re
import sys
import textwrap
Expand All @@ -27,6 +28,24 @@ def locate_wheels(base_dir):
wheels = glob.glob(os.path.join(base_dir, '*.whl'))
return sorted(wheels)

def parse_req(req):
try:
req_object = Requirement.parse(req)
req_name = req_object.key
spec = str(req_object).replace(req_name, '')
return (req_name, spec)
except:
print('Failed to parse requirement %s' % (req))

def record_dep(dependencies, req, lib_name):
req_name, spec = parse_req(req)
if not req_name in dependencies:
dependencies[req_name] = {}
if not spec in dependencies[req_name]:
dependencies[req_name][spec] = []
dependencies[req_name][spec].append(lib_name)


def get_lib_deps(base_dir):
packages = {}
dependencies = {}
Expand All @@ -44,15 +63,7 @@ def get_lib_deps(base_dir):
}

for req in requires:
req_parts = re.split('([<>~=]+)', req, 1)
req_name = req_parts[0]
spec = ''.join(req_parts[1:])
spec = ','.join(sorted(spec.split(',')))
if not req_name in dependencies:
dependencies[req_name] = {}
if not spec in dependencies[req_name]:
dependencies[req_name][spec] = []
dependencies[req_name][spec].append(lib_name)
record_dep(dependencies, req, lib_name)
except:
print('Failed to parse %s' % (setup_path))
return packages, dependencies
Expand All @@ -61,7 +72,6 @@ def get_wheel_deps(wheel_dir):
from wheel.pkginfo import read_pkg_info_bytes
from wheel.wheelfile import WheelFile

requires_dist_re = re.compile(r"""^(?P<name>\S+)(\s\((?P<spec>.+)\))?$""")
packages = {}
dependencies = {}
for whl_path in locate_wheels(wheel_dir):
Expand All @@ -79,14 +89,8 @@ def get_wheel_deps(wheel_dir):

requires = pkg_info.get_all('Requires-Dist')
for req in requires:
parsed = requires_dist_re.match(req.split(';')[0].strip())
req_name, spec = parsed.group('name', 'spec')
spec = ','.join(sorted(spec.split(','))) if spec else ''
if not req_name in dependencies:
dependencies[req_name] = {}
if not spec in dependencies[req_name]:
dependencies[req_name][spec] = []
dependencies[req_name][spec].append(lib_name)
req = re.sub(r'[\s\(\)]', '', req)
record_dep(dependencies, req, lib_name)
except:
print('Failed to parse METADATA from %s' % (whl_path))
return packages, dependencies
Expand Down Expand Up @@ -241,9 +245,7 @@ def render_report(output_path, report_context):
try:
with open(frozen_filename, 'r') as frozen_file:
for line in frozen_file:
req_parts = re.split('([<>~=]+)', line.strip(), 1)
req_name = req_parts[0]
spec = ''.join(req_parts[1:])
req_name, spec = parse_req(line)
frozen[req_name] = [spec]
except:
print('Unable to open shared_requirements.txt, shared requirements have not been validated')
Expand Down Expand Up @@ -307,4 +309,4 @@ def display_order(k):
elif not args.verbose:
print('Library dependencies do not match frozen requirements, run this script with --verbose for details')

sys.exit(exitcode)
sys.exit(exitcode)