Show every Discourse post w/ excerpt

app/discourse/models.py

@@ -1,5 +1,5 @@
 import datetime
-from sqlalchemy import types, Column, ForeignKey
+from sqlalchemy import types, Column, ForeignKey, UniqueConstraint
 
 from ..models import db, User
 from .. import models
@@ -15,12 +15,25 @@ def parse_iso_datetime(text):
 
     return datetime.datetime.strptime(text, "%Y-%m-%dT%H:%M:%S.%fZ")
 
+def rebase_hrefs(html, origin=None):
+    '''
+    >>> rebase_hrefs(u'<a href="/blarg">hi</a>', 'http://bop')
+    u'<a href="http://bop/blarg">hi</a>'
+    '''
+
+    if origin is None:
+        origin = config.origin
+
+    return html.replace(u'href="/', u'href="%s/' % unicode(origin))
+
 class DiscourseTopicEvent(models.UserEvent):
     __tablename__ = 'discourse_topics'
 
     id = Column(types.Integer, ForeignKey('user_events.id'), primary_key=True)
 
-    discourse_id = Column(types.Integer, unique=True)
+    discourse_id = Column(types.Integer)
+
+    post_number = Column(types.Integer)
 
     slug = Column(types.Text)
 
@@ -34,52 +47,72 @@ class DiscourseTopicEvent(models.UserEvent):
 
     category_slug = Column(types.Text)
 
+    __table_args__ = (UniqueConstraint('discourse_id', 'post_number'),)
+
     __mapper_args__ = {
         'polymorphic_identity': 'discourse_topic_event'
     }
 
+    @property
+    def cleaned_excerpt(self):
+        return rebase_hrefs(self.excerpt or '')
+
     @property
     def url(self):
-        return config.url('/t/%s/%d' % (self.slug, self.discourse_id))
+        return config.url('/t/%s/%d/%d' % (self.slug, self.discourse_id,
+                                           self.post_number or 0))
 
     @property
     def category_url(self):
         return config.url('/c/%s' % self.category_slug)
 
     @classmethod
-    def _get_or_create(cls, discourse_id):
+    def _get_or_create(cls, discourse_id, post_number=None):
         msg = db.session.query(cls).\
-              filter_by(discourse_id=discourse_id).first()
+              filter_by(discourse_id=discourse_id,
+                        post_number=post_number).first()
         if msg is None:
-            msg = cls(discourse_id=discourse_id)
+            msg = cls(discourse_id=discourse_id, post_number=post_number)
         return msg
 
     @classmethod
-    def _update_category(cls, category):
-        topics = category.get('topics', [])
-        for topic in topics:
-            if not topic['visible']: continue
-            msg = cls._get_or_create(discourse_id=topic['id'])
-            msg.created_at = parse_iso_datetime(topic['created_at'])
-            msg.updated_at = parse_iso_datetime(topic['bumped_at'])
+    def _update_topic(cls, category, topic):
+        req = api.get('/t/%d/last.json' % topic['id'])
+
+        if req.status_code != 200:
+            return req.raise_for_status()
+
+        topic_detail = req.json()
+
+        for post in topic_detail['post_stream']['posts']:
+            if post['hidden'] or not post['cooked']: continue
+
+            msg = cls._get_or_create(discourse_id=topic['id'],
+                                     post_number=post['post_number'])
+            msg.created_at = parse_iso_datetime(post['created_at'])
+            msg.updated_at = parse_iso_datetime(post['updated_at'])
             msg.slug = topic['slug']
 
             msg.category_name = category['name']
             msg.category_slug = category['slug']
 
-            user = User.find_by_username(topic['last_poster']['username'])
+            user = User.find_by_username(post['username'])
             msg.user = user
 
-            # Argh, it looks like only pinned topics have excerpts
-            # for now:
-            #
-            # https://meta.discourse.org/t/get-excerpt-for-regular-topics/33482
-            msg.excerpt = topic.get('excerpt')
+            msg.excerpt = post['cooked']
 
             msg.title = topic['title']
             msg.posts_count = topic['posts_count']
             db.session.add(msg)
 
+    @classmethod
+    def _update_category(cls, category):
+        topics = category.get('topics', [])
+        for topic in topics:
+            if not topic['visible']: continue
+
+            cls._update_topic(category, topic)
+
     @classmethod
     def update(cls):
         req = api.get('/categories.json')

app/templates/_activity_events.html

@@ -38,16 +38,12 @@
         {% if event.category_name %}<a href="{{ event.category_url }}" class="e-discourse-category">{{ event.category_name }}</a>{% endif %}
       </p>
       {% if event.excerpt %}
-      <p>{{ event.excerpt|safe }}</p>
+      <p>{{ event.cleaned_excerpt|safe }}</p>
       {% endif %}
       <div class="e-actions">
         <a href="{{ event.url }}"><span class="material-icons">open_in_new</span>
         <span class="e-action-label">
-          {% if event.posts_count > 1 %}
-            {{ gettext("Read %(posts_count)s replies on Discuss", posts_count=event.posts_count) }}
-          {% else %}
-            {{ gettext("Read more on Discuss") }}
-          {% endif %}
+          {{ gettext("Read more on Discuss") }}
         </span></a>
       </div>
     </div>

app/tests/test_discourse.py

@@ -193,13 +193,23 @@ class DiscourseTopicEventTests(DbTestCase):
     BASE_APP_CONFIG.update(DISCOURSE=FAKE_DISCOURSE_CONFIG)
 
     def test_url_works(self):
+        evt = DiscourseTopicEvent(discourse_id=5, post_number=1,
+                                  slug='beep-boop')
+        self.assertEqual(evt.url, 'http://discourse/t/beep-boop/5/1')
+
+    def test_url_works_with_no_post_number(self):
         evt = DiscourseTopicEvent(discourse_id=5, slug='beep-boop')
-        self.assertEqual(evt.url, 'http://discourse/t/beep-boop/5')
+        self.assertEqual(evt.url, 'http://discourse/t/beep-boop/5/0')
 
     def test_category_url_works(self):
         evt = DiscourseTopicEvent(category_slug='beep-boop')
         self.assertEqual(evt.category_url, 'http://discourse/c/beep-boop')
 
+    def test_cleaned_excerpt_works(self):
+        evt = DiscourseTopicEvent(excerpt='<a href="/blah">hi</a>')
+        self.assertEqual(evt.cleaned_excerpt,
+                         '<a href="http://discourse/blah">hi</a>')
+
     @mock.patch('app.discourse.api.get')
     def test_update_works(self, get):
         fake_topics = [
@@ -210,12 +220,7 @@ def test_update_works(self, get):
             {
                 'visible': True,
                 'id': 14,
-                'bumped_at': '2016-02-18T14:27:48.103Z',
-                'created_at': '2016-02-15T14:27:48.062Z',
                 'posts_count': 6,
-                'last_poster': {
-                    'username': 'system'
-                },
                 'title': 'Hello There',
                 'slug': 'hello-there',
             }
@@ -236,14 +241,39 @@ def test_update_works(self, get):
                 ]
             }
         }
+        fake_post = {
+            'hidden': False,
+            'cooked': '<p>Hello</p>',
+            'post_number': 1,
+            'created_at': '2016-02-15T14:27:48.062Z',
+            'updated_at': '2016-02-18T14:27:48.103Z',
+            'username': 'system',
+        }
+        fake_topic_detail = {
+            'post_stream': {
+                'posts': [fake_post]
+            }
+        }
 
-        get.return_value.status_code = 200
-        get.return_value.json.return_value = fake_categories
+        def get_url(url):
+            retval = mock.MagicMock()
+            retval.raise_for_status.side_effect = Exception('kaboom')
+            if url == '/categories.json':
+                retval.status_code = 200
+                retval.json.return_value = fake_categories
+            else:
+                retval.status_code = 200
+                retval.json.return_value = fake_topic_detail
+            return retval
+
+        get.side_effect = get_url
 
         DiscourseTopicEvent.update()
 
-        get.assert_called_once_with('/categories.json')
-        get.return_value.raise_for_status.assert_not_called()
+        get.assert_has_calls([
+            mock.call('/categories.json'),
+            mock.call('/t/14/last.json')
+        ])
 
         events = db.session.query(Event).all()
         self.assertEqual(len(events), 1)
@@ -257,23 +287,23 @@ def test_update_works(self, get):
                          datetime.datetime(2016, 2, 18, 14, 27, 48, 103000))
         self.assertEqual(event.slug, 'hello-there')
         self.assertIsNone(event.user)
-        self.assertIsNone(event.excerpt)
+        self.assertEqual(event.excerpt, '<p>Hello</p>')
         self.assertEqual(event.title, 'Hello There')
         self.assertEqual(event.category_name, 'Funky Things')
         self.assertEqual(event.category_slug, 'funky-things')
         self.assertEqual(event.posts_count, 6)
 
-        # Now simulate a new reply.
+        # Now simulate an edited post.
 
-        fake_topics[1]['bumped_at'] = '2016-02-20T14:27:48.103Z'
-        fake_topics[1]['posts_count'] += 1
+        fake_post['updated_at'] = '2016-02-20T14:27:48.103Z'
+        fake_post['cooked'] = '<p>Blah</p>'
 
         DiscourseTopicEvent.update()
 
         self.assertEqual(db.session.query(Event).all(), [event])
         self.assertEqual(event.updated_at,
                          datetime.datetime(2016, 2, 20, 14, 27, 48, 103000))
-        self.assertEqual(event.posts_count, 7)
+        self.assertEqual(event.excerpt, '<p>Blah</p>')
 
     def test_get_or_create_can_get_existing(self):
         evt = DiscourseTopicEvent(discourse_id=15)

migrations/versions/28ff85fcc188_.py

@@ -0,0 +1,26 @@
+"""add post_number column to discourse_topics
+
+Revision ID: 28ff85fcc188
+Revises: 312a48268855
+Create Date: 2016-03-22 22:00:07.520355
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '28ff85fcc188'
+down_revision = '312a48268855'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('discourse_topics', sa.Column('post_number', sa.Integer(), nullable=True))
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('discourse_topics', 'post_number')
+    ### end Alembic commands ###

migrations/versions/51bdb7c7928a_.py

@@ -0,0 +1,28 @@
+"""enforce uniqueness based on discourse_id and post_number
+
+Revision ID: 51bdb7c7928a
+Revises: 28ff85fcc188
+Create Date: 2016-03-22 22:02:14.557660
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '51bdb7c7928a'
+down_revision = '28ff85fcc188'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(u'discourse_topics_discourse_id_key', 'discourse_topics', type_='unique')
+    op.create_unique_constraint(None, 'discourse_topics', ['discourse_id', 'post_number'])
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'discourse_topics', type_='unique')
+    op.create_unique_constraint(u'discourse_topics_discourse_id_key', 'discourse_topics', ['discourse_id'])
+    ### end Alembic commands ###