From 65d07e6b4084540c3d454db590d40456f117e52e Mon Sep 17 00:00:00 2001 From: Atul Varma Date: Tue, 22 Mar 2016 18:40:46 -0400 Subject: [PATCH 1/3] Show every Discourse post w/ excerpt. Fixes #301. --- app/discourse/models.py | 56 +++++++++++++++++++---------- app/templates/_activity_events.html | 6 +--- app/tests/test_discourse.py | 55 ++++++++++++++++++++-------- 3 files changed, 78 insertions(+), 39 deletions(-) diff --git a/app/discourse/models.py b/app/discourse/models.py index 69439d0..dd5901d 100644 --- a/app/discourse/models.py +++ b/app/discourse/models.py @@ -1,5 +1,5 @@ import datetime -from sqlalchemy import types, Column, ForeignKey +from sqlalchemy import types, Column, ForeignKey, UniqueConstraint from ..models import db, User from .. import models @@ -20,7 +20,9 @@ class DiscourseTopicEvent(models.UserEvent): id = Column(types.Integer, ForeignKey('user_events.id'), primary_key=True) - discourse_id = Column(types.Integer, unique=True) + discourse_id = Column(types.Integer) + + post_number = Column(types.Integer) slug = Column(types.Text) @@ -34,52 +36,68 @@ class DiscourseTopicEvent(models.UserEvent): category_slug = Column(types.Text) + __table_args__ = (UniqueConstraint('discourse_id', 'post_number'),) + __mapper_args__ = { 'polymorphic_identity': 'discourse_topic_event' } @property def url(self): - return config.url('/t/%s/%d' % (self.slug, self.discourse_id)) + return config.url('/t/%s/%d/%d' % (self.slug, self.discourse_id, + self.post_number or 0)) @property def category_url(self): return config.url('/c/%s' % self.category_slug) @classmethod - def _get_or_create(cls, discourse_id): + def _get_or_create(cls, discourse_id, post_number=None): msg = db.session.query(cls).\ - filter_by(discourse_id=discourse_id).first() + filter_by(discourse_id=discourse_id, + post_number=post_number).first() if msg is None: - msg = cls(discourse_id=discourse_id) + msg = cls(discourse_id=discourse_id, post_number=post_number) return msg @classmethod - def _update_category(cls, category): - topics = category.get('topics', []) - for topic in topics: - if not topic['visible']: continue - msg = cls._get_or_create(discourse_id=topic['id']) - msg.created_at = parse_iso_datetime(topic['created_at']) - msg.updated_at = parse_iso_datetime(topic['bumped_at']) + def _update_topic(cls, category, topic): + req = api.get('/t/%d/last.json' % topic['id']) + + if req.status_code != 200: + return req.raise_for_status() + + topic_detail = req.json() + + for post in topic_detail['post_stream']['posts']: + if post['hidden'] or not post['cooked']: continue + + msg = cls._get_or_create(discourse_id=topic['id'], + post_number=post['post_number']) + msg.created_at = parse_iso_datetime(post['created_at']) + msg.updated_at = parse_iso_datetime(post['updated_at']) msg.slug = topic['slug'] msg.category_name = category['name'] msg.category_slug = category['slug'] - user = User.find_by_username(topic['last_poster']['username']) + user = User.find_by_username(post['username']) msg.user = user - # Argh, it looks like only pinned topics have excerpts - # for now: - # - # https://meta.discourse.org/t/get-excerpt-for-regular-topics/33482 - msg.excerpt = topic.get('excerpt') + msg.excerpt = post['cooked'] msg.title = topic['title'] msg.posts_count = topic['posts_count'] db.session.add(msg) + @classmethod + def _update_category(cls, category): + topics = category.get('topics', []) + for topic in topics: + if not topic['visible']: continue + + cls._update_topic(category, topic) + @classmethod def update(cls): req = api.get('/categories.json') diff --git a/app/templates/_activity_events.html b/app/templates/_activity_events.html index 3be8bb8..609a534 100644 --- a/app/templates/_activity_events.html +++ b/app/templates/_activity_events.html @@ -43,11 +43,7 @@
open_in_new - {% if event.posts_count > 1 %} - {{ gettext("Read %(posts_count)s replies on Discuss", posts_count=event.posts_count) }} - {% else %} - {{ gettext("Read more on Discuss") }} - {% endif %} + {{ gettext("Read more on Discuss") }}
diff --git a/app/tests/test_discourse.py b/app/tests/test_discourse.py index 03e0989..7dafc9f 100644 --- a/app/tests/test_discourse.py +++ b/app/tests/test_discourse.py @@ -193,8 +193,13 @@ class DiscourseTopicEventTests(DbTestCase): BASE_APP_CONFIG.update(DISCOURSE=FAKE_DISCOURSE_CONFIG) def test_url_works(self): + evt = DiscourseTopicEvent(discourse_id=5, post_number=1, + slug='beep-boop') + self.assertEqual(evt.url, 'http://discourse/t/beep-boop/5/1') + + def test_url_works_with_no_post_number(self): evt = DiscourseTopicEvent(discourse_id=5, slug='beep-boop') - self.assertEqual(evt.url, 'http://discourse/t/beep-boop/5') + self.assertEqual(evt.url, 'http://discourse/t/beep-boop/5/0') def test_category_url_works(self): evt = DiscourseTopicEvent(category_slug='beep-boop') @@ -210,12 +215,7 @@ def test_update_works(self, get): { 'visible': True, 'id': 14, - 'bumped_at': '2016-02-18T14:27:48.103Z', - 'created_at': '2016-02-15T14:27:48.062Z', 'posts_count': 6, - 'last_poster': { - 'username': 'system' - }, 'title': 'Hello There', 'slug': 'hello-there', } @@ -236,14 +236,39 @@ def test_update_works(self, get): ] } } + fake_post = { + 'hidden': False, + 'cooked': '

Hello

', + 'post_number': 1, + 'created_at': '2016-02-15T14:27:48.062Z', + 'updated_at': '2016-02-18T14:27:48.103Z', + 'username': 'system', + } + fake_topic_detail = { + 'post_stream': { + 'posts': [fake_post] + } + } - get.return_value.status_code = 200 - get.return_value.json.return_value = fake_categories + def get_url(url): + retval = mock.MagicMock() + retval.raise_for_status.side_effect = Exception('kaboom') + if url == '/categories.json': + retval.status_code = 200 + retval.json.return_value = fake_categories + else: + retval.status_code = 200 + retval.json.return_value = fake_topic_detail + return retval + + get.side_effect = get_url DiscourseTopicEvent.update() - get.assert_called_once_with('/categories.json') - get.return_value.raise_for_status.assert_not_called() + get.assert_has_calls([ + mock.call('/categories.json'), + mock.call('/t/14/last.json') + ]) events = db.session.query(Event).all() self.assertEqual(len(events), 1) @@ -257,23 +282,23 @@ def test_update_works(self, get): datetime.datetime(2016, 2, 18, 14, 27, 48, 103000)) self.assertEqual(event.slug, 'hello-there') self.assertIsNone(event.user) - self.assertIsNone(event.excerpt) + self.assertEqual(event.excerpt, '

Hello

') self.assertEqual(event.title, 'Hello There') self.assertEqual(event.category_name, 'Funky Things') self.assertEqual(event.category_slug, 'funky-things') self.assertEqual(event.posts_count, 6) - # Now simulate a new reply. + # Now simulate an edited post. - fake_topics[1]['bumped_at'] = '2016-02-20T14:27:48.103Z' - fake_topics[1]['posts_count'] += 1 + fake_post['updated_at'] = '2016-02-20T14:27:48.103Z' + fake_post['cooked'] = '

Blah

' DiscourseTopicEvent.update() self.assertEqual(db.session.query(Event).all(), [event]) self.assertEqual(event.updated_at, datetime.datetime(2016, 2, 20, 14, 27, 48, 103000)) - self.assertEqual(event.posts_count, 7) + self.assertEqual(event.excerpt, '

Blah

') def test_get_or_create_can_get_existing(self): evt = DiscourseTopicEvent(discourse_id=15) From 426fc5f68533c216524428f33f05aa6a1824ccda Mon Sep 17 00:00:00 2001 From: Atul Varma Date: Wed, 23 Mar 2016 06:56:54 -0400 Subject: [PATCH 2/3] rebase hrefs from discourse snippets. --- app/discourse/models.py | 15 +++++++++++++++ app/templates/_activity_events.html | 2 +- app/tests/test_discourse.py | 5 +++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/app/discourse/models.py b/app/discourse/models.py index dd5901d..f36192b 100644 --- a/app/discourse/models.py +++ b/app/discourse/models.py @@ -15,6 +15,17 @@ def parse_iso_datetime(text): return datetime.datetime.strptime(text, "%Y-%m-%dT%H:%M:%S.%fZ") +def rebase_hrefs(html, origin=None): + ''' + >>> rebase_hrefs(u'hi', 'http://bop') + u'hi' + ''' + + if origin is None: + origin = config.origin + + return html.replace(u'href="/', u'href="%s/' % unicode(origin)) + class DiscourseTopicEvent(models.UserEvent): __tablename__ = 'discourse_topics' @@ -42,6 +53,10 @@ class DiscourseTopicEvent(models.UserEvent): 'polymorphic_identity': 'discourse_topic_event' } + @property + def cleaned_excerpt(self): + return rebase_hrefs(self.excerpt or '') + @property def url(self): return config.url('/t/%s/%d/%d' % (self.slug, self.discourse_id, diff --git a/app/templates/_activity_events.html b/app/templates/_activity_events.html index 609a534..a3f8007 100644 --- a/app/templates/_activity_events.html +++ b/app/templates/_activity_events.html @@ -38,7 +38,7 @@ {% if event.category_name %}{{ event.category_name }}{% endif %}

{% if event.excerpt %} -

{{ event.excerpt|safe }}

+

{{ event.cleaned_excerpt|safe }}

{% endif %}
open_in_new diff --git a/app/tests/test_discourse.py b/app/tests/test_discourse.py index 7dafc9f..d9424cc 100644 --- a/app/tests/test_discourse.py +++ b/app/tests/test_discourse.py @@ -205,6 +205,11 @@ def test_category_url_works(self): evt = DiscourseTopicEvent(category_slug='beep-boop') self.assertEqual(evt.category_url, 'http://discourse/c/beep-boop') + def test_cleaned_excerpt_works(self): + evt = DiscourseTopicEvent(excerpt='hi') + self.assertEqual(evt.cleaned_excerpt, + 'hi') + @mock.patch('app.discourse.api.get') def test_update_works(self, get): fake_topics = [ From cada612e6b86289cf3621f09d7637ef0c21ed03d Mon Sep 17 00:00:00 2001 From: Atul Varma Date: Wed, 23 Mar 2016 07:02:22 -0400 Subject: [PATCH 3/3] Add migrations for discourse_topics changes. --- migrations/versions/28ff85fcc188_.py | 26 ++++++++++++++++++++++++++ migrations/versions/51bdb7c7928a_.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 migrations/versions/28ff85fcc188_.py create mode 100644 migrations/versions/51bdb7c7928a_.py diff --git a/migrations/versions/28ff85fcc188_.py b/migrations/versions/28ff85fcc188_.py new file mode 100644 index 0000000..4605969 --- /dev/null +++ b/migrations/versions/28ff85fcc188_.py @@ -0,0 +1,26 @@ +"""add post_number column to discourse_topics + +Revision ID: 28ff85fcc188 +Revises: 312a48268855 +Create Date: 2016-03-22 22:00:07.520355 + +""" + +# revision identifiers, used by Alembic. +revision = '28ff85fcc188' +down_revision = '312a48268855' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('discourse_topics', sa.Column('post_number', sa.Integer(), nullable=True)) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('discourse_topics', 'post_number') + ### end Alembic commands ### diff --git a/migrations/versions/51bdb7c7928a_.py b/migrations/versions/51bdb7c7928a_.py new file mode 100644 index 0000000..2c15093 --- /dev/null +++ b/migrations/versions/51bdb7c7928a_.py @@ -0,0 +1,28 @@ +"""enforce uniqueness based on discourse_id and post_number + +Revision ID: 51bdb7c7928a +Revises: 28ff85fcc188 +Create Date: 2016-03-22 22:02:14.557660 + +""" + +# revision identifiers, used by Alembic. +revision = '51bdb7c7928a' +down_revision = '28ff85fcc188' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(u'discourse_topics_discourse_id_key', 'discourse_topics', type_='unique') + op.create_unique_constraint(None, 'discourse_topics', ['discourse_id', 'post_number']) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'discourse_topics', type_='unique') + op.create_unique_constraint(u'discourse_topics_discourse_id_key', 'discourse_topics', ['discourse_id']) + ### end Alembic commands ###