Add language_code to DB schema, and populate with languageCode from API response. (#195)

* Add language_code to DB schema, and populate with languageCode from API response.

* add missing, line terminating comma in namedtuple field list

* add langauge_code column name to insert statement

Author: macpd

crowdtangle/db_functions.py

@@ -25,13 +25,13 @@ def upsert_posts(self, posts):
             '%(title)s,%(platform)s, %(platform_id)s, %(post_url)s, %(subscriber_count)s, '
             '%(type)s, %(updated)s, %(video_length_ms)s, %(image_text)s, %(legacy_id)s, '
             '%(caption)s, %(link)s, %(date)s, %(description)s, %(score)s, %(live_video_status)s, '
-            'CURRENT_TIMESTAMP)')
+            '%(language_code)s, CURRENT_TIMESTAMP)')
         insert_posts_query = (
             '''INSERT INTO posts(id, account_id, branded_content_sponsor_account_id, message, title,
             platform, platform_id, post_url, subscriber_count, type, updated, video_length_ms,
             image_text, legacy_id, caption, link, date, description, score, live_video_status,
-            last_modified_time) VALUES %s ON CONFLICT (id) DO UPDATE SET id = EXCLUDED.id,
-            account_id = EXCLUDED.account_id,
+            language_code, last_modified_time) VALUES %s
+            ON CONFLICT (id) DO UPDATE SET id = EXCLUDED.id, account_id = EXCLUDED.account_id,
             branded_content_sponsor_account_id = EXCLUDED.branded_content_sponsor_account_id,
             message = EXCLUDED.message, title = EXCLUDED.title, platform = EXCLUDED.platform,
             platform_id = EXCLUDED.platform_id, post_url = EXCLUDED.post_url,
@@ -40,7 +40,8 @@ def upsert_posts(self, posts):
             image_text = EXCLUDED.image_text, legacy_id = EXCLUDED.legacy_id,
             caption = EXCLUDED.caption, link = EXCLUDED.link, date = EXCLUDED.date,
             description = EXCLUDED.description, score = EXCLUDED.score,
-            live_video_status = EXCLUDED.live_video_status, last_modified_time = CURRENT_TIMESTAMP
+            live_video_status = EXCLUDED.live_video_status, language_code = EXCLUDED.language_code,
+            last_modified_time = CURRENT_TIMESTAMP
             WHERE posts.id = EXCLUDED.id AND posts.updated < EXCLUDED.updated;''')
         psycopg2.extras.execute_values(cursor,
                                        insert_posts_query,

crowdtangle/process_crowdtangle_posts.py

@@ -36,7 +36,8 @@
                          'date',
                          'description',
                          'score',
-                         'live_video_status'])
+                         'live_video_status',
+                         'language_code'])
 AccountRecord = namedtuple('AccountRecord',
                            ['id',
                             'account_type',
@@ -141,7 +142,8 @@ def process(self, item):
             date=item.get('date'),
             description=item.get('description'),
             score=item.get('score'),
-            live_video_status=item.get('liveVideoStatus'))
+            live_video_status=item.get('liveVideoStatus'),
+            language_code=item.get('languageCode'))
 
         account_list = [self.make_account_record(item['account'], post_updated)]
 

sql/unified_schema.sql

@@ -342,6 +342,7 @@ CREATE TABLE public.posts (
   description character varying,
   score double precision,
   live_video_status character varying,
+  language_code character varying,
   last_modified_time timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
   CONSTRAINT account_id_fk FOREIGN KEY (account_id) REFERENCES public.accounts (id) MATCH SIMPLE ON UPDATE NO ACTION ON DELETE NO ACTION,
   CONSTRAINT branded_content_sponsor_account_id_fk FOREIGN KEY (branded_content_sponsor_account_id) REFERENCES public.accounts (id) MATCH SIMPLE ON UPDATE NO ACTION ON DELETE NO ACTION