Skip to content
This repository was archived by the owner on Feb 26, 2025. It is now read-only.
Merged
Changes from 1 commit
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
e431c32
dafsa etld_datacreated successfully
heyitsarpit May 31, 2019
482e864
Replaced wget dependency and added kinto record publishing code
heyitsarpit Jun 3, 2019
4403331
errors for fetching functions handled with decorator
heyitsarpit Jun 3, 2019
2d798e1
kinto update records
heyitsarpit Jun 5, 2019
de83ba2
script refactored, publish_dafsa now entry point for script
heyitsarpit Jun 5, 2019
a26a443
refactored redundant nested else statement
heyitsarpit Jun 5, 2019
3f27425
get_latest_hash() now only returns hash when public_suffix_list.dat i…
heyitsarpit Jun 5, 2019
f41bef1
removed directory as **kwargs argument
heyitsarpit Jun 5, 2019
c845e6a
Credentials replaced with environment variables
heyitsarpit Jun 5, 2019
0c5e08a
attachment uploading
heyitsarpit Jun 5, 2019
b2175e4
Creating filepaths with os.path.join()
heyitsarpit Jun 5, 2019
06fbcd6
removed mimitypes module, check respose status, initialised client wi…
heyitsarpit Jun 6, 2019
024f261
get_latest_hash() function now checks for just the single file with n…
heyitsarpit Jun 6, 2019
9bc4dbc
Record and attachment now uploaded in the single session
heyitsarpit Jun 7, 2019
9aa0843
Client creation moved to different function for now.
heyitsarpit Jun 7, 2019
14f0e3a
regrouped imports and removed decorator function
heyitsarpit Jun 10, 2019
8171561
raising more exceptions and getting server/auth from event object
heyitsarpit Jun 10, 2019
361d5a7
Record fetching to compare and passing client as argument
heyitsarpit Jun 11, 2019
f76a857
added auth split to tuple
heyitsarpit Jun 11, 2019
386f6d7
removed redundant code and refactoring
heyitsarpit Jun 11, 2019
104de46
auth key and dafsa flag added
heyitsarpit Jun 11, 2019
a7a27ec
get bucket through env, added context arg to entry function
heyitsarpit Jun 12, 2019
af955d8
conform to flake8 standard
heyitsarpit Jun 12, 2019
022cff4
conform to black standard
heyitsarpit Jun 12, 2019
1cce889
Request review
heyitsarpit Jun 12, 2019
09c92f3
refactor
heyitsarpit Jun 12, 2019
6f9a0c2
dafsa creation comment update
heyitsarpit Jun 12, 2019
70fc4a1
tests added for publish_dafsa and added argument for get_latest_hash()
heyitsarpit Jun 13, 2019
9740e66
Test added for dafsa creation
heyitsarpit Jun 19, 2019
dd45bca
dafsa file creation now captures output of stdout
heyitsarpit Jun 20, 2019
3ab6aff
kinto client tests
heyitsarpit Jun 20, 2019
fb0fb61
dafsa creation and publishing broken into two methods
heyitsarpit Jun 21, 2019
179e730
tests for new functions
heyitsarpit Jun 24, 2019
9d798a5
tests wip
heyitsarpit Jun 27, 2019
1faccfa
kinto expection test
heyitsarpit Jun 27, 2019
9e68bef
All assertRaises as context managers
heyitsarpit Jun 27, 2019
b3b541e
get_latest_hash test passing
heyitsarpit Jun 27, 2019
2f16062
removed unneccsasry responses and added new test
heyitsarpit Jun 28, 2019
a751efd
new function for record fetching
heyitsarpit Jun 28, 2019
e018a64
tests refactored
heyitsarpit Jun 30, 2019
8822fe4
all tests passing except remote settings publish
heyitsarpit Jun 30, 2019
a2c6eae
added decortor to test_HTTPError_raised_when_404
heyitsarpit Jun 30, 2019
3350c70
test for remote_settings_publish now passing
heyitsarpit Jun 30, 2019
4cca805
removed redundant bits and added a new test
heyitsarpit Jul 1, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Creating filepaths with os.path.join()
  • Loading branch information
heyitsarpit committed Jun 5, 2019
commit b2175e4b456fa7fa64c79eceec970b5f396f3860
31 changes: 17 additions & 14 deletions commands/publish_dafsa.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ def get_latest_hash():
def download_resources(directory, *urls):
for url in urls:
# file_location is found by appending the file_name(at the end of url string) to temp directory
file_location = directory + "/" + url.split("/")[-1]
file_name = os.path.basename(url)
file_location = os.path.join(directory, file_name)
print(file_location)
response = requests.get(url, stream=True)
with open(file_location, "wb") as f:
Expand All @@ -75,15 +76,12 @@ def make_dafsa_and_publish(client, latest_hash):
prepare_tlds.py is called with the two arguments the location of
the downloaded public suffix list and the name of the output file
"""
output_filepath = f"{tmp}/etld_data.json"
prepare_tlds_py_path = os.path.join(tmp, "prepare_tlds.py")
raw_psl_path = os.path.join(tmp, "public_suffix_list.dat")
output_binary_path = os.path.join(tmp, "etld_data.json")

subprocess.run(
[
"python3",
f"{tmp}/prepare_tlds.py",
f"{tmp}/public_suffix_list.dat",
output_filepath,
]
["python3", prepare_tlds_py_path, raw_psl_path, output_binary_path]
)

subprocess.run(["ls", tmp])
Expand All @@ -95,11 +93,11 @@ def make_dafsa_and_publish(client, latest_hash):
collection=COLLECTION_ID,
bucket=BUCKET_ID,
)

# Upload the attachment
mimetype, _ = mimetypes.guess_type(output_filepath)
filename = os.path.basename(output_filepath)
filecontent = open(output_filepath, "rb").read()
mimetype, _ = mimetypes.guess_type(output_binary_path)
filename = os.path.basename(output_binary_path)
filecontent = open(output_binary_path, "rb").read()
record_uri = client.get_endpoint(
"record", id=RECORD_ID, bucket=BUCKET_ID, collection=COLLECTION_ID
)
Expand All @@ -110,15 +108,20 @@ def make_dafsa_and_publish(client, latest_hash):
method="post", endpoint=attachment_uri, files=multipart
)
except KintoException as e:
print(output_filepath, "error during upload.", e)
print(output_binary_path, "error during upload.", e)
else:
print(body)


def publish_dafsa():
client = Client(server_url=SERVER, auth=CREDENTIALS)
latest_hash = get_latest_hash()
record = client.get_record(id=RECORD_ID, bucket=BUCKET_ID, collection=COLLECTION_ID)
try:
record = client.get_record(
id=RECORD_ID, bucket=BUCKET_ID, collection=COLLECTION_ID
)
except KintoException as e:
print(e)
if record_exists(record) and (record["data"]["latest-commit-hash"] == latest_hash):
return 0
else:
Expand Down