diff --git a/poetry.lock b/poetry.lock index 0e680fe..b732c4f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -299,6 +299,40 @@ sql-other = ["SQLAlchemy (>=1.4.36)"] test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.8.0)"] +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + [[package]] name = "python-dateutil" version = "2.8.2" @@ -335,6 +369,17 @@ files = [ {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, ] +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + [[package]] name = "requests" version = "2.31.0" @@ -423,4 +468,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "d905e0dd67c69848986c8fe249615451678a299e88a1b9ab71ddee5226a67e48" +content-hash = "36c1aeafffd28eddfb29d4cc381b23a4e6a9a04ff64d97eea00e723f0a35b2fc" diff --git a/pyproject.toml b/pyproject.toml index 63e15b8..5869c8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ pandas = "^2.1.0" python-decouple = "^3.8" hugchat = "^0.2.2" dependencies = "^7.7.0" +pendulum = "^2.1.2" [tool.poetry.group.dev.dependencies] diff --git a/src/repositories/scrapper.py b/src/repositories/scrapper.py index 2a31c5d..a0e372c 100644 --- a/src/repositories/scrapper.py +++ b/src/repositories/scrapper.py @@ -2,6 +2,7 @@ from typing import Any import pandas as pd +import pendulum import requests from repositories.texts_database import TextsDatabaseRepository @@ -33,7 +34,7 @@ def __init__(self, scrapper_email: str, scrapper_password: str) -> None: self.token = response.json()["accessToken"] self.biography_id = response.json()["projectsList"][0]["_id"] - # self.userId = response.json()["user"]["_id"] + self.userId = response.json()["user"]["_id"] def post(self, url: str, data: dict[str, Any]) -> requests.Response: return self.__session.post( @@ -51,6 +52,13 @@ class Scrapper: TRANSCRIPT_URL = ( "https://redaction-backend-prod.herokuapp.com/platform/transcripts/get-model" ) + MASTER_URL = ( + "https://redaction-backend-prod.herokuapp.com/platform/masters/get-model" + ) + CREATE_STORY = ( + "https://redaction-backend-prod.herokuapp.com/platform/chapters/create-new" + ) + SAVE_TEXT_URL = "https://redaction-backend-prod.herokuapp.com/platform/masters/save-text-modifications" def __init__( self, @@ -60,29 +68,6 @@ def __init__( self.__connection = entoureo_connection self.__texts_database = texts_database - # def login(self) -> None: - # if not self.__email or not self.__password: - # print( - # f"Found {self.__email} for username and {self.__password} " - # f"for password. Need to set them in the .env file" - # ) - # return - - # payload = {"email": self.__email, "password": self.__password} - - # print("Login to entoureo ...") - # response = self.__session.post(self.LOGIN_URL, data=payload) - # assert ( - # 200 <= response.status_code < 300 - # ), f"The request wasn't valid, got status code {response.status_code}" - # print( - # response.status_code - # ) # If the request went Ok we usually get a 200 status. - - # self.token = response.json()["accessToken"] - # self.biography_id = response.json()["projectsList"][0]["_id"] - # self.userId = response.json()["user"]["_id"] - def get_all_chapters(self) -> pd.DataFrame: payload = {"biographyId": self.__connection.biography_id} @@ -160,3 +145,52 @@ def get_transcription( transcriptName = response.json()["transcript"]["title"] return (transcriptName, transcriptData) + + def create_new_story( + self, + name: str, + ): + response = self.__connection.post( + url=self.CREATE_STORY, + data={ + "biographyId": self.__connection.biography_id, + "userId": self.__connection.userId, + "newChapterTitle": name, + }, + ) + print("Response story creation: ", response.status_code) + + def save_to_story(self, chapter_id: str, text: str): + # first step get what they call the "master" info + response = self.__connection.post( + url=self.MASTER_URL, + data={ + "chapterId": chapter_id, + "userId": self.__connection.userId, + }, + ) + assert ( + 200 <= response.status_code < 300 + ), f"The request wasn't valid, got status code {response.status_code}" + print("Response code: ", response.status_code) + master = response.json()["master"] + master["currentText"] = text + # date using this format: 2021-09-28T14:00:00.000Z + # master["dateOfLastModification"] = pendulum.now("UTC").to_iso8601_string() + import json + + print(json.dumps(master, indent=4)) + # second step save the text + response = self.__connection.post( + url=self.SAVE_TEXT_URL, + data={ + "chapterId": chapter_id, + "userId": self.__connection.userId, + "master": master, + }, + ) + assert ( + 200 <= response.status_code < 300 + ), f"The request wasn't valid, got status code {response.status_code}" + print("Response code: ", response.status_code) + print(response.json()) diff --git a/src/scraping_test_script.py b/src/scraping_test_script.py index c430945..1e0d60f 100644 --- a/src/scraping_test_script.py +++ b/src/scraping_test_script.py @@ -2,14 +2,15 @@ def main(): - chapters = Container.scrapper.get_all_chapters() - transcriptions = Container.scrapper.get_chapter_data(chapters.loc[0, "_id"]) + # chapters = Container.scrapper.get_all_chapters() + # transcriptions = Container.scrapper.get_chapter_data(chapters.loc[0, "_id"]) - transcriptName, transcriptData = Container.scrapper.get_transcription( - transcriptions[0]["_id"], chapters.loc[0, "_id"] - ) - print(transcriptName) - print(transcriptData) + # transcriptName, transcriptData = Container.scrapper.get_transcription( + # transcriptions[0]["_id"], chapters.loc[0, "_id"] + # ) + # print(transcriptName) + # print(transcriptData) + Container.scrapper.save_to_story("6529456e77d0b130fde581ac", "rwrite this shit") if __name__ == "__main__":