diff --git a/.env.sample b/.env.sample index 7ee4e66..8bd7e12 100644 --- a/.env.sample +++ b/.env.sample @@ -6,8 +6,13 @@ LOCAL_TIMEZONE=US/Pacific GOOGLE_CLIENT_ID= GOOGLE_CLIENT_SECRET= + CANVAS_SERVER_URL= CANVAS_CLIENT_ID= CANVAS_CLIENT_SECRET= CANVAS_COURSE_ID= -SENDGRID_API_KEY= + +EMAIL_SERVER= +EMAIL_PORT= +EMAIL_USE_TLS= +EMAIL_USERNAME= diff --git a/README.md b/README.md index 1aaccd6..4052a1b 100644 --- a/README.md +++ b/README.md @@ -105,27 +105,7 @@ Student and email data could come from different sources (Google Sheets, csv upl #### Emailing students -(Under Construction; content might be outdated) - -Students will receive an email that looks like - -``` -Hi -name-, - -Here's your assigned seat for -exam-: - -Room: -room- - -Seat: -seat- - -You can view this seat's position on the seating chart at: -/seat/-seatid-/ - --additional text- -``` - -The "additional text" is a good place to tell them what to do if they have an -issue with their seat, and to sign the email (e.g. "- Cal CS 61A Staff"). +In the emailing page, you should specify minimally sender's address and sender's signature. The signature is appended to the end of the email body. You can also specify additional text to be appended to the end of the email body (this is a good place to tell students what to do if they have concerns about their seating assignments). The email body is generated from a template. The template is a html file that you can edit. The template is located at `server/services/email/templates/assignment_inform_email.html` and its variables are automatically filled in. #### Get Roster Photos @@ -231,11 +211,14 @@ CANVAS_SERVER_URL= CANVAS_CLIENT_ID= CANVAS_CLIENT_SECRET= -# sendgrid api key, get it from sendgrid dashboard -SENDGRID_API_KEY= +# email setup, use any smtp server +EMAIL_SERVER= +EMAIL_PORT= +EMAIL_USE_TLS= +EMAIL_USERNAME= # misc -DOMAIN=localhost:5000 +SERVER_BASE_URL=localhost:5000 LOCAL_TIMEZONE=US/Pacific ``` diff --git a/config.py b/config.py index 27dd57a..4972a4e 100644 --- a/config.py +++ b/config.py @@ -35,10 +35,13 @@ def getenv(key, default: Optional[str] = None): MOCK_CANVAS = getenv('MOCK_CANVAS', 'false').lower() == 'true' SEND_EMAIL = getenv('SEND_EMAIL', 'off').lower() - # Email setup. Domain environment is for link in email. - SENDGRID_API_KEY = getenv('SENDGRID_API_KEY', "placeholder") + # Email setup. + EMAIL_SERVER = getenv('EMAIL_SERVER', "unset") + EMAIL_PORT = getenv('EMAIL_PORT', "unset") + EMAIL_USERNAME = getenv('EMAIL_USERNAME', "unset") + EMAIL_PASSWORD = getenv('EMAIL_PASSWORD', "unset") - PHOTO_DIRECTORY = getenv('PHOTO_DIRECTORY', "placeholder") + PHOTO_DIRECTORY = getenv('PHOTO_DIRECTORY', "unset") class ProductionConfig(ConfigBase): diff --git a/requirements-dev.txt b/requirements-dev.txt index 9997feb..f2ed407 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,5 @@ safety==1.10.3 pip-audit==2.6.1 selenium==4.14.0 responses==0.23.3 +aiosmtpd==1.4.4.post2 flask-fixtures==0.3.8 diff --git a/requirements.txt b/requirements.txt index 0237e53..3e5059c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ Flask_WTF==1.1.1 google_api_python_client==2.101.0 natsort==8.4.0 oauth2client==4.1.3 -sendgrid==6.10.0 SQLAlchemy==1.4.42 Werkzeug==0.16.1 WTForms==3.0.1 diff --git a/server/forms.py b/server/forms.py index a8bcf9e..4ba9cf6 100644 --- a/server/forms.py +++ b/server/forms.py @@ -83,11 +83,11 @@ class AssignForm(FlaskForm): class EmailForm(FlaskForm): - from_email = StringField('from_email', [Email()]) - from_name = StringField('from_name', [InputRequired()]) - subject = StringField('subject', [InputRequired()]) - test_email = StringField('test_email') - additional_text = TextAreaField('additional_text') + from_addr = StringField('from_addr', [Email(), InputRequired()]) + signature = StringField('signature', [InputRequired()]) + additional_info = TextAreaField('additional_info') + override_subject = StringField('override_subject') + override_to_addr = StringField('override_to_addr') submit = SubmitField('send') diff --git a/server/models.py b/server/models.py index 9bf89d0..dd1c514 100644 --- a/server/models.py +++ b/server/models.py @@ -84,6 +84,18 @@ def unassigned_seats(self): def unassigned_students(self): return [student for student in self.students if student.assignment == None] # noqa + def get_assignments(self, emailed=None, limit=None, offset=None): + query = SeatAssignment.query.join(SeatAssignment.seat).join(Seat.room).filter( + Room.exam_id == self.id, + ) + if emailed is not None: + query = query.filter(SeatAssignment.emailed == emailed) + if limit is not None: + query = query.limit(limit) + if offset is not None: + query = query.offset(offset) + return query.all() + def __repr__(self): return ''.format(self.name) diff --git a/server/services/email/__init__.py b/server/services/email/__init__.py index 9fb8321..5daf238 100644 --- a/server/services/email/__init__.py +++ b/server/services/email/__init__.py @@ -1,73 +1,73 @@ +from server import app +from server.models import db, Exam, SeatAssignment, Offering +from server.services.email.smtp import SMTPConfig, send_email +import server.services.email.templates as templates +from server.typings.enum import EmailTemplate +from flask import url_for +import os -import sendgrid +_email_config = SMTPConfig( + app.config.get('EMAIL_SERVER'), + app.config.get('EMAIL_PORT'), + app.config.get('EMAIL_USERNAME'), + app.config.get('EMAIL_PASSWORD') +) -from server import app -from server.models import db, Room, Seat, SeatAssignment +def email_students(exam: Exam, form): + ASSIGNMENT_PER_PAGE = 500 + page_number = 1 + emailed_count = 0 -def email_students(exam, form): - """Emails students in batches of 900""" - sg = sendgrid.SendGridAPIClient(api_key=app.config['SENDGRID_API_KEY']) - test = form.test_email.data while True: - limit = 1 if test else 900 - assignments = SeatAssignment.query.join(SeatAssignment.seat).join(Seat.room).filter( - Room.exam_id == exam.id, - not SeatAssignment.emailed - ).limit(limit).all() + assignments = exam.get_assignments( + emailed=False, + limit=ASSIGNMENT_PER_PAGE, + offset=(page_number - 1) * ASSIGNMENT_PER_PAGE + ) if not assignments: break + page_number += 1 - data = { - 'personalizations': [ - { - 'to': [ - { - 'email': test if test else assignment.student.email, - } - ], - 'substitutions': { - '-name-': assignment.student.first_name, - '-room-': assignment.seat.room.display_name, - '-seat-': assignment.seat.name, - '-seatid-': str(assignment.seat.id), - }, - } - for assignment in assignments - ], - 'from': { - 'email': form.from_email.data, - 'name': form.from_name.data, - }, - 'subject': form.subject.data, - 'content': [ - { - 'type': 'text/plain', - 'value': ''' -Hi -name-, + for assignment in assignments: + result = _email_single_assignment(exam.offering, exam, assignment, form) + if result[0]: + emailed_count += 1 + assignment.emailed = True + else: + db.session.commit() + return result + else: + db.session.commit() -Here's your assigned seat for {}: + if emailed_count == 0: + return (False, "No unemailed assignments found.") -Room: -room- + return (True, ) -Seat: -seat- -You can view this seat's position on the seating chart at: -{}/seat/-seatid-/ +def _email_single_assignment(offering: Offering, exam: Exam, assignment: SeatAssignment, form) -> bool: + seat_path = url_for('student_single_seat', seat_id=assignment.seat.id) + seat_absolute_path = os.path.join(app.config.get('SERVER_BASE_URL'), seat_path) -{} -'''.format(exam.display_name, app.config['DOMAIN'], form.additional_text.data) - }, - ], - } + student_email = \ + templates.get_email(EmailTemplate.ASSIGNMENT_INFORM_EMAIL, + {"EXAM": exam.display_name}, + {"NAME": assignment.student.first_name, + "COURSE": offering.name, + "EXAM": exam.display_name, + "ROOM": assignment.seat.room.display_name, + "SEAT": assignment.seat.name, + "URL": seat_absolute_path, + "ADDITIONAL_INFO": form.additional_info.data, + "SIGNATURE": form.signature.data}) - response = sg.client.mail.send.post(request_body=data) - if response.status_code < 200 or response.status_code >= 400: - raise Exception('Could not send mail. Status: {}. Body: {}'.format( - response.status_code, response.body - )) - if test: - return - for assignment in assignments: - assignment.emailed = True - db.session.commit() + effective_to_addr = form.override_to_addr.data or assignment.student.email + effective_subject = form.override_subject.data or student_email.subject + + return send_email(smtp=_email_config, + from_addr=form.from_addr.data, + to_addr=effective_to_addr, + subject=effective_subject, + body=student_email.body, + body_html=student_email.body if student_email.body_html else None) diff --git a/server/services/email/smtp.py b/server/services/email/smtp.py new file mode 100644 index 0000000..6d173f3 --- /dev/null +++ b/server/services/email/smtp.py @@ -0,0 +1,40 @@ +from smtplib import SMTP +from email.message import EmailMessage + + +class SMTPConfig: + def __init__(self, smtp_server, smtp_port, username, password, use_tls=True, use_auth=True): + self.smtp_server = smtp_server + self.smtp_port = smtp_port + self.username = username + self.password = password + self.use_tls = use_tls + self.use_auth = use_auth + + def __repr__(self) -> str: + return f'SMTPConfig(smtp_server={self.smtp_server}, smtp_port={self.smtp_port}, username={self.username}, use_tls={self.use_tls}, use_auth={self.use_auth})' # noqa + + +def send_email(*, smtp: SMTPConfig, from_addr, to_addr, subject, body, body_html=None): + msg = EmailMessage() + msg['From'] = from_addr + msg['To'] = to_addr + msg['Subject'] = subject + msg.set_content(body) + + if body_html: + msg.add_alternative(body_html, subtype='html') + + try: + server = SMTP(smtp.smtp_server, smtp.smtp_port) + # if server.has_extn('STARTTLS'): + if smtp.use_tls: + server.starttls() + # if server.has_extn('AUTH'): + if smtp.use_auth: + server.login(smtp.username, smtp.password) + server.send_message(msg) + server.quit() + return (True, ) + except Exception as e: + return (False, f"Error sending email: {e.message}\n Config: \n{smtp}") diff --git a/server/services/email/templates/__init__.py b/server/services/email/templates/__init__.py new file mode 100644 index 0000000..5619507 --- /dev/null +++ b/server/services/email/templates/__init__.py @@ -0,0 +1,40 @@ +from server.typings.enum import EmailTemplate +import os +import json +import re + + +class EmailContent: + def __init__(self, subject: str, body: str, body_html: bool): + self.subject = subject + self.body = body + self.body_html = body_html + + def __repr__(self) -> str: + return f'EmailContent(subject={self.subject}, body={self.body}, body_type={self.body_html})' + + +def get_email( + template: EmailTemplate, + subject_substitutions: dict[str, str], + body_substitutions: dict[str, str] +) -> EmailContent: + # template.value is the file name for email metadata (json) + # let us read it that first. The file sits in the same path as this file. + email_metadata = None + with open(os.path.join(os.path.dirname(__file__), template.value + ".json")) as f: + email_metadata = json.load(f) + subject = _make_substitutions(email_metadata['subject'], subject_substitutions) + body_html = email_metadata['body_html'] + body_path = os.path.join(os.path.dirname(__file__), email_metadata['body_path']) + body = None + with open(body_path) as f: + body = _make_substitutions(f.read(), body_substitutions) + content = EmailContent(subject, body, re.match('[Tt]rue', body_html) is not None) + return content + + +def _make_substitutions(text: str, substitutions: dict[str, str]) -> str: + for key, value in substitutions.items(): + text = re.sub(r'\{\{\s*' + key + r'\s*\}\}', value, text) + return text diff --git a/server/services/email/templates/assignment_inform_email.html b/server/services/email/templates/assignment_inform_email.html new file mode 100644 index 0000000..23ba9cb --- /dev/null +++ b/server/services/email/templates/assignment_inform_email.html @@ -0,0 +1,33 @@ + + + + Seating Assignment for {{COURSE}} {{EXAM}} + + +

Dear {{NAME}},

+ +

Here's your assigned seat for the upcoming exam:

+ + + +

You can view this seat's position on the seating chart at:

+ View Seating Chart + +

+ If you have any questions or require further assistance, please do not + hesitate to contact the relevant course staff. +

+ +

Good luck on your exam!

+ +

{{ADDITIONAL_INFO}}

+ +

Best,

+

{{SIGNATURE}}

+ + diff --git a/server/services/email/templates/assignment_inform_email.json b/server/services/email/templates/assignment_inform_email.json new file mode 100644 index 0000000..d9e001b --- /dev/null +++ b/server/services/email/templates/assignment_inform_email.json @@ -0,0 +1,5 @@ +{ + "subject": "Your Exam Seating Assignment for {{EXAM}}", + "body_path": "assignment_inform_email.html", + "body_html": "true" +} diff --git a/server/templates/email.html.j2 b/server/templates/email.html.j2 index 391249e..5623a0a 100644 --- a/server/templates/email.html.j2 +++ b/server/templates/email.html.j2 @@ -7,35 +7,32 @@
From
- {{ form.from_email(class="mdl-textfield__input", type="Email", id="fromemail") }} - + {{ form.from_addr(class="mdl-textfield__input", type="Email", id="from_addr") }} +
- {{ form.from_name(class="mdl-textfield__input", type="Name", id="fromname") }} - + {{ form.signature(class="mdl-textfield__input", type="Name", id="signature") }} +
-
-
Subject
+
+
Additional Info
- {{ form.subject(class="mdl-textfield__input", type="Subject", id="subject") }} - + {{ form.additional_info(class="mdl-textfield__input", type="text", rows="10", id="additional_info") }} +
-
-
-
Test Email
+
+
+
Overrides
- {{ form.test_email(class="mdl-textfield__input", type="Test Email", id="testemail") }} - + {{ form.override_subject(class="mdl-textfield__input", type="Subject", id="subject") }} +
-
-
-
Additional Text
- {{ form.additional_text(class="mdl-textfield__input", type="text", rows="10", id="addtext") }} - + {{ form.override_to_addr(class="mdl-textfield__input", type="Test Email", id="testemail") }} +
-
+
{{ form.submit(class="mdl-button mdl-js-button mdl-button--raised") }}
diff --git a/server/typings/enum.py b/server/typings/enum.py index 309cae0..bc658a8 100644 --- a/server/typings/enum.py +++ b/server/typings/enum.py @@ -12,3 +12,7 @@ class EmailSendingConfig(Enum): OFF = 'off' # does not send emails TEST = 'test' # sends all emails to test email address ON = 'on' # sends emails to real email addresses + + +class EmailTemplate(Enum): + ASSIGNMENT_INFORM_EMAIL = 'assignment_inform_email' diff --git a/server/views.py b/server/views.py index 1efadf0..8d8b991 100644 --- a/server/views.py +++ b/server/views.py @@ -330,7 +330,9 @@ def assign(exam): def email(exam): form = EmailForm() if form.validate_on_submit(): - email_students(exam, form) + success, payload = email_students(exam, form) + if not success: + return payload return redirect(url_for('students', exam=exam)) return render_template('email.html.j2', exam=exam, form=form) # endregion diff --git a/tests/unit/test_email.py b/tests/unit/test_email.py new file mode 100644 index 0000000..f0061ba --- /dev/null +++ b/tests/unit/test_email.py @@ -0,0 +1,194 @@ +import pytest +from unittest.mock import patch +import server.services.email.templates as templates +from server.services.email import send_email, _email_config, SMTPConfig +from server.typings.enum import EmailTemplate +from email.message import Message + +TEST_FROM_EMAIL = 'sender@example.com' +TEST_TO_EMAIL = 'recipient@example.com' +TEST_SUBJECT = 'Test Subject' +TEST_BODY = 'Test Body' +TEST_BODY_HTML = '

Test Body

' + + +def _get_content(msg: Message, type='text/html'): + """ + returns the content of the email body with the given type + """ + if msg.is_multipart(): + for part in msg.get_payload(): + if part.get_content_type() == type: + return part.get_payload(decode=True).decode(part.get_content_charset()) + else: + if msg.get_content_type() == type: + return msg.get_payload(decode=True).decode(msg.get_content_charset()) + return None + + +@patch('server.services.email.smtp.SMTP') +def test_send_plain_text_email(mock_smtp): + """ + Stubs out the SMTP server and checks that plain text email is sent correctly + """ + + success = send_email(smtp=_email_config, + from_addr=TEST_FROM_EMAIL, + to_addr=TEST_TO_EMAIL, + subject=TEST_SUBJECT, + body=TEST_BODY) + + assert success[0] + + # check the use of smtp server + mock_smtp.assert_called_with(_email_config.smtp_server, _email_config.smtp_port) + mock_smtp.return_value.starttls.assert_called_once() + mock_smtp.return_value.login.assert_called_once_with(_email_config.username, _email_config.password) + mock_smtp.return_value.send_message.assert_called_once() + mock_smtp.return_value.quit.assert_called_once() + + # check email meta + msg = mock_smtp.return_value.send_message.call_args[0][0] + assert msg['From'] == TEST_FROM_EMAIL + assert msg['To'] == TEST_TO_EMAIL + assert msg['Subject'] == TEST_SUBJECT + + # check plain text content + assert TEST_BODY in msg.get_payload() + + +@patch('server.services.email.smtp.SMTP') +def test_send_html_email(mock_smtp): + """ + Stubs out the SMTP server and checks that html email is sent correctly + """ + + success = send_email(smtp=_email_config, + from_addr=TEST_FROM_EMAIL, + to_addr=TEST_TO_EMAIL, + subject=TEST_SUBJECT, + body=TEST_BODY, + body_html=TEST_BODY_HTML) + + assert success[0] + + msg = mock_smtp.return_value.send_message.call_args[0][0] + html = _get_content(msg, 'text/html') + assert html is not None + assert TEST_BODY_HTML in html + + +import threading # noqa +from aiosmtpd.controller import Controller # noqa +from aiosmtpd.handlers import Message as MessageHandler # noqa +from email import message_from_string # noqa + +_fake_email_config = SMTPConfig('127.0.0.1', 1025, 'user', 'pass', use_tls=False, use_auth=False) + + +class CustomMessageHandler(MessageHandler): + received_message = [] + + def handle_message(self, message): + CustomMessageHandler.received_message.append(message_from_string(message.as_string())) + + +@pytest.fixture() +def smtp_server(): + controller = Controller(CustomMessageHandler(), + hostname=_fake_email_config.smtp_server, + port=_fake_email_config.smtp_port) + # has to use 127.0.0.1 instead of localhost so that the test can run on Github Actions + # otherwise, the test does not seem to be able to find the smtp server + thread = threading.Thread(target=controller.start) + thread.start() + + yield controller + + CustomMessageHandler.received_message = [] + controller.stop() + thread.join() + + +def test_send_plain_text_email_with_mock_smtp_server(smtp_server): + """ + Use a local fake smtp server to test that plain text email is sent correctly + """ + + success = send_email( + smtp=_fake_email_config, + from_addr=TEST_FROM_EMAIL, + to_addr=TEST_TO_EMAIL, + subject=TEST_SUBJECT, + body=TEST_BODY) + + assert success[0] + + msg = CustomMessageHandler.received_message[0] + + assert msg is not None + assert msg['From'] == TEST_FROM_EMAIL + assert msg['To'] == TEST_TO_EMAIL + assert msg['Subject'] == TEST_SUBJECT + assert TEST_BODY in msg.get_payload() + + +def test_send_html_email_with_mock_smtp_server(smtp_server): + """ + Use a local fake smtp server to test that html email is sent correctly + """ + + success = send_email( + smtp=_fake_email_config, + from_addr=TEST_FROM_EMAIL, + to_addr=TEST_TO_EMAIL, + subject=TEST_SUBJECT, + body=TEST_BODY, + body_html=TEST_BODY_HTML) + + assert success[0] + + msg = CustomMessageHandler.received_message[0] + + # check html content + html = _get_content(msg, 'text/html') + assert html is not None + assert TEST_BODY_HTML in html + + +def test_send_seating_html_email_with_mock_smtp_server(smtp_server): + + test_seating_email = \ + templates.get_email(EmailTemplate.ASSIGNMENT_INFORM_EMAIL, + {"EXAM": "test exam"}, + {"NAME": "test name", + "COURSE": "test course", + "EXAM": "test exam", + "ROOM": "test room", + "SEAT": "test seat", + "URL": "test/url", + "ADDITIONAL_INFO": "test additional text", + "SIGNATURE": "test signature"}) + + success = send_email(smtp=_fake_email_config, + from_addr=TEST_FROM_EMAIL, + to_addr=TEST_TO_EMAIL, + subject=test_seating_email.subject, + body=test_seating_email.body, + body_html=test_seating_email.body if test_seating_email.body_html else None) + + assert success[0] + + msg = CustomMessageHandler.received_message[0] + + assert msg['From'] == TEST_FROM_EMAIL + assert msg['To'] == TEST_TO_EMAIL + assert msg['Subject'] == test_seating_email.subject + + html = _get_content(msg, 'text/html') + assert html is not None + assert test_seating_email.body in html + + +def test_send_email_for_exam_with_mock_smtp_server(smtp_server): + pass