From 26e08b39b39babe7202967e5bd3ae1e4b8d4a9aa Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Sat, 31 Aug 2019 00:09:09 +0600
Subject: [PATCH 01/17] Store deb packages list for a device.

---
 backend/backend/settings/dev.py               |  2 +-
 backend/device_registry/api_views.py          | 15 +++++++++++----
 .../migrations/0056_device_deb_packages.py    | 19 +++++++++++++++++++
 backend/device_registry/models.py             |  1 +
 4 files changed, 32 insertions(+), 5 deletions(-)
 create mode 100644 backend/device_registry/migrations/0056_device_deb_packages.py

diff --git a/backend/backend/settings/dev.py b/backend/backend/settings/dev.py
index 09b969fa1..78fa67a90 100644
--- a/backend/backend/settings/dev.py
+++ b/backend/backend/settings/dev.py
@@ -12,7 +12,7 @@
         'PORT': os.getenv('DB_PORT', '5432'),
         'OPTIONS': {
             'connect_timeout': 3,
-        },
+        }
     }
 }
 COMMON_NAME_PREFIX = 'd.wott-dev.local'
diff --git a/backend/device_registry/api_views.py b/backend/device_registry/api_views.py
index 334728c3e..a9279b19a 100644
--- a/backend/device_registry/api_views.py
+++ b/backend/device_registry/api_views.py
@@ -58,15 +58,22 @@ def get(self, request, *args, **kwargs):
         firewallstate_object, _ = FirewallState.objects.get_or_create(device=device)
         block_networks = portscan_object.block_networks.copy()
         block_networks.extend(settings.SPAM_NETWORKS)
-        return Response({'policy': firewallstate_object.policy_string,
-                         firewallstate_object.ports_field_name: portscan_object.block_ports,
-                         'block_networks': block_networks})
+        return Response({
+            'firewall': {
+                'policy': firewallstate_object.policy_string,
+                firewallstate_object.ports_field_name: portscan_object.block_ports,
+                'block_networks': block_networks
+            },
+            'deb_packages_hash': device.deb_packages.get('hash')
+        })
 
     def post(self, request, *args, **kwargs):
         data = request.data
         device = Device.objects.get(device_id=request.device_id)
         device.last_ping = timezone.now()
         device.agent_version = data.get('agent_version')
+        if 'deb_packages' in data:
+            device.deb_packages = data['deb_packages']
 
         device_info_object, _ = DeviceInfo.objects.get_or_create(device=device)
         device_info_object.device__last_ping = timezone.now()
@@ -102,7 +109,7 @@ def post(self, request, *args, **kwargs):
         firewall_state.rules = firewall_rules
         firewall_state.save()
 
-        device.save(update_fields=['last_ping', 'agent_version', 'trust_score'])
+        device.save(update_fields=['last_ping', 'agent_version', 'deb_packages', 'trust_score'])
 
         if datastore_client:
             task_key = datastore_client.key('Ping')
diff --git a/backend/device_registry/migrations/0056_device_deb_packages.py b/backend/device_registry/migrations/0056_device_deb_packages.py
new file mode 100644
index 000000000..32a2b9a43
--- /dev/null
+++ b/backend/device_registry/migrations/0056_device_deb_packages.py
@@ -0,0 +1,19 @@
+# Generated by Django 2.1.10 on 2019-08-30 10:48
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('device_registry', '0055_remove_deviceinfo_trust_score'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='device',
+            name='deb_packages',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict),
+        ),
+    ]
diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py
index 918430de0..1353a9491 100644
--- a/backend/device_registry/models.py
+++ b/backend/device_registry/models.py
@@ -65,6 +65,7 @@ class Device(models.Model):
     agent_version = models.CharField(max_length=36, blank=True, null=True)
     tags = tagulous.models.TagField(to=Tag, blank=True)
     trust_score = models.FloatField(null=True)
+    deb_packages = JSONField(blank=True, default=dict)
 
     @property
     def certificate_expired(self):

From 1f375a619afcece1ed1c06b547cd904eb3c73817 Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Sat, 31 Aug 2019 11:15:12 +0600
Subject: [PATCH 02/17] Fixing tests.

---
 .../0054_device_calculated_trust_score.py     |  2 +-
 backend/device_registry/tests/test_api.py     | 30 ++++++++++++++-----
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/backend/device_registry/migrations/0054_device_calculated_trust_score.py b/backend/device_registry/migrations/0054_device_calculated_trust_score.py
index add0931b7..569900de3 100644
--- a/backend/device_registry/migrations/0054_device_calculated_trust_score.py
+++ b/backend/device_registry/migrations/0054_device_calculated_trust_score.py
@@ -1,10 +1,10 @@
 # Generated by Django 2.1.10 on 2019-08-06 05:10
 
 from django.db import migrations, models
-from device_registry.models import Device
 
 
 def save_trust_score(apps, schema_editor):
+    Device = apps.get_model('device_registry', 'Device')
     for d in Device.objects.all():
         d.save(update_fields=['trust_score'])
 
diff --git a/backend/device_registry/tests/test_api.py b/backend/device_registry/tests/test_api.py
index a732a9d79..40d4e4283 100644
--- a/backend/device_registry/tests/test_api.py
+++ b/backend/device_registry/tests/test_api.py
@@ -717,15 +717,26 @@ def setUp(self):
     def test_ping_get_success(self):
         response = self.client.get(self.url, **self.headers)
         self.assertEqual(response.status_code, 200)
-        self.assertDictEqual(response.data, {'policy': self.device.firewallstate.policy_string,
-                                             'block_ports': [], 'block_networks': settings.SPAM_NETWORKS})
+        self.assertDictEqual(response.data, {
+            'firewall': {
+                'policy': self.device.firewallstate.policy_string,
+                'block_ports': [], 'block_networks': settings.SPAM_NETWORKS
+            },
+            'deb_packages_hash': None
+        })
 
     def test_pong_data(self):
         # 1st request
         response = self.client.get(self.url, **self.headers)
         self.assertEqual(response.status_code, 200)
-        self.assertDictEqual(response.data, {'block_ports': [], 'block_networks': settings.SPAM_NETWORKS,
-                                             'policy': self.device.firewallstate.policy_string})
+        self.assertDictEqual(response.data, {
+            'firewall': {
+                'block_ports': [], 
+                'block_networks': settings.SPAM_NETWORKS,
+                'policy': self.device.firewallstate.policy_string
+            },
+            'deb_packages_hash': None
+        })
         # 2nd request
         self.device.portscan.block_ports = [['192.168.1.178', 'tcp', 22, False]]
         self.device.portscan.block_networks = [['192.168.1.177', False]]
@@ -735,9 +746,14 @@ def test_pong_data(self):
 
         response = self.client.get(self.url, **self.headers)
         self.assertEqual(response.status_code, 200)
-        self.assertDictEqual(response.data, {'policy': self.device.firewallstate.policy_string,
-                                             'block_ports': [['192.168.1.178', 'tcp', 22, False]],
-                                             'block_networks': [['192.168.1.177', False]] + settings.SPAM_NETWORKS})
+        self.assertDictEqual(response.data, {
+            'firewall': {
+                'policy': self.device.firewallstate.policy_string,
+                'block_ports': [['192.168.1.178', 'tcp', 22, False]],
+                'block_networks': [['192.168.1.177', False]] + settings.SPAM_NETWORKS
+            },
+            'deb_packages_hash': None
+        })
 
     def test_ping_creates_models(self):
         devinfo_obj_count_before = DeviceInfo.objects.count()

From 2b156df01503897a58eda915bc1959dfa474158e Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Mon, 2 Sep 2019 18:43:33 +0600
Subject: [PATCH 03/17] Warn about insecure services #391

---
 backend/device_registry/models.py               | 17 +++++++++++++++++
 .../templates/device_info_security.html         |  6 +++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py
index 1353a9491..5a95c3177 100644
--- a/backend/device_registry/models.py
+++ b/backend/device_registry/models.py
@@ -71,6 +71,23 @@ class Device(models.Model):
     def certificate_expired(self):
         return self.certificate_expires < timezone.now()
 
+    INSECURE_SERVICES = [
+        'fingerd',
+        'tftpd',
+        'telnetd',
+        'snmpd',
+        'xinetd',
+        'nis',
+        'atftpd',
+        'tftpd-hpa',
+        'rsh-server',
+        'rsh-redone-server'
+    ]
+    @property
+    def insecure_services(self):
+        packages = set([p['name'] for p in self.deb_packages['packages']])
+        return set(self.INSECURE_SERVICES) & packages
+
     def get_name(self):
         if self.name:
             return self.name
diff --git a/backend/device_registry/templates/device_info_security.html b/backend/device_registry/templates/device_info_security.html
index f7605eb77..10f0f0cf0 100644
--- a/backend/device_registry/templates/device_info_security.html
+++ b/backend/device_registry/templates/device_info_security.html
@@ -78,7 +78,11 @@ <h4 class="tab-title">Security</h4>
                     <th scope="row">
                         Insecure Services
                     </th>
-                  <td>Coming soon!</td>
+                  <td>
+                  {% for service_name in object.insecure_services %}
+                    <p class="m-0">{{ service_name }}</p>
+                  {% endfor %}
+                  </td>
                 </tr>
                 <th scope="row">Logins</th>
                 <td>

From eba44cce514501b4709e0c43d588c86983df7f0b Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Mon, 2 Sep 2019 18:51:16 +0600
Subject: [PATCH 04/17] Handle case when deb_packages is empty.

---
 backend/device_registry/models.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py
index 5a95c3177..a5cf4f51d 100644
--- a/backend/device_registry/models.py
+++ b/backend/device_registry/models.py
@@ -85,8 +85,9 @@ def certificate_expired(self):
     ]
     @property
     def insecure_services(self):
-        packages = set([p['name'] for p in self.deb_packages['packages']])
-        return set(self.INSECURE_SERVICES) & packages
+        if 'packages' in self.deb_packages:
+            packages = set([p['name'] for p in self.deb_packages['packages']])
+            return set(self.INSECURE_SERVICES) & packages
 
     def get_name(self):
         if self.name:

From c8b3e7bc9b8183bcdc6c40575f10aec793a13426 Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Mon, 2 Sep 2019 22:51:11 +0600
Subject: [PATCH 05/17] Add test for "insecure services"

---
 .../templates/device_info_security.html       |  4 +++-
 backend/device_registry/tests/test_all.py     | 20 ++++++++++++++++++
 backend/device_registry/tests/test_api.py     | 21 +++++++++++++++++++
 3 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/backend/device_registry/templates/device_info_security.html b/backend/device_registry/templates/device_info_security.html
index 10f0f0cf0..3928cf5d2 100644
--- a/backend/device_registry/templates/device_info_security.html
+++ b/backend/device_registry/templates/device_info_security.html
@@ -79,9 +79,11 @@ <h4 class="tab-title">Security</h4>
                         Insecure Services
                     </th>
                   <td>
+                  <ul>
                   {% for service_name in object.insecure_services %}
-                    <p class="m-0">{{ service_name }}</p>
+                    <li>{{ service_name }}</p>
                   {% endfor %}
+                  </ul>
                   </td>
                 </tr>
                 <th scope="row">Logins</th>
diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py
index ce88b193f..6f414f6e5 100644
--- a/backend/device_registry/tests/test_all.py
+++ b/backend/device_registry/tests/test_all.py
@@ -532,6 +532,26 @@ def test_logins(self):
         self.assertEqual(response.status_code, 200)
         self.assertContains(response, '<pre>pi:')
         self.assertContains(response, 'success: 1')
+    
+    def test_insecure_services(self):
+        self.client.login(username='test', password='123')
+        url = reverse('device-detail-security', kwargs={'pk': self.device.pk})
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertNotContains(response, '>telnetd<')
+        self.assertNotContains(response, '>fingerd<')
+
+        self.device.deb_packages = {
+            'packages': [
+                {'name': 'telnetd', 'version': 'VERSION'},
+                {'name': 'fingerd', 'version': 'VERSION'}
+            ]
+        }
+        self.device.save()
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, '>telnetd<')
+        self.assertContains(response, '>fingerd<')
 
 
 class PairingKeysView(TestCase):
diff --git a/backend/device_registry/tests/test_api.py b/backend/device_registry/tests/test_api.py
index 40d4e4283..3d5865e12 100644
--- a/backend/device_registry/tests/test_api.py
+++ b/backend/device_registry/tests/test_api.py
@@ -849,6 +849,27 @@ def test_ping_writes_trust_score(self):
         self.device.refresh_from_db()
         self.assertGreater(self.device.trust_score, 0.42)
 
+    def test_ping_writes_packages(self):
+        packages = [{'name': 'PACKAGE', 'version': 'VERSION'}]
+        self.ping_payload['deb_packages'] = {
+            'hash': 'abcdef',
+            'packages': packages
+        }
+        self.client.post(self.url, self.ping_payload, **self.headers)
+
+        response = self.client.get(self.url, **self.headers)
+        self.assertEqual(response.status_code, 200)
+        self.assertDictEqual(response.data, {
+            'firewall': {
+                'block_ports': [],
+                'block_networks': settings.SPAM_NETWORKS,
+                'policy': self.device.firewallstate.policy_string
+            },
+            'deb_packages_hash': 'abcdef'
+        })
+        self.device.refresh_from_db()
+        self.assertListEqual(self.device.deb_packages, packages)
+
 
 class DeviceEnrollView(APITestCase):
     def setUp(self):

From 8484534ab8a0aab86ed36df3a4f81c7ad97b7caf Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Mon, 2 Sep 2019 22:59:43 +0600
Subject: [PATCH 06/17] Add "no insecure services detected" message. Fix
 test_ping_writes_packages.

---
 .../templates/device_info_security.html          | 16 +++++++++++-----
 backend/device_registry/tests/test_all.py        |  1 +
 backend/device_registry/tests/test_api.py        |  2 +-
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/backend/device_registry/templates/device_info_security.html b/backend/device_registry/templates/device_info_security.html
index 3928cf5d2..ad957d5eb 100644
--- a/backend/device_registry/templates/device_info_security.html
+++ b/backend/device_registry/templates/device_info_security.html
@@ -79,11 +79,17 @@ <h4 class="tab-title">Security</h4>
                         Insecure Services
                     </th>
                   <td>
-                  <ul>
-                  {% for service_name in object.insecure_services %}
-                    <li>{{ service_name }}</p>
-                  {% endfor %}
-                  </ul>
+                  {% with object.insecure_services as services %}
+                    {% if services %}
+                    <ul>
+                    {% for service_name in services %}
+                      <li>{{ service_name }}</p>
+                    {% endfor %}
+                    </ul>
+                    {% else %}
+                    No insecure services detected.
+                    {% endif %}
+                  {% endwith %}
                   </td>
                 </tr>
                 <th scope="row">Logins</th>
diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py
index 6f414f6e5..c0c01ac55 100644
--- a/backend/device_registry/tests/test_all.py
+++ b/backend/device_registry/tests/test_all.py
@@ -540,6 +540,7 @@ def test_insecure_services(self):
         self.assertEqual(response.status_code, 200)
         self.assertNotContains(response, '>telnetd<')
         self.assertNotContains(response, '>fingerd<')
+        self.assertContains(response, 'No insecure services detected')
 
         self.device.deb_packages = {
             'packages': [
diff --git a/backend/device_registry/tests/test_api.py b/backend/device_registry/tests/test_api.py
index 3d5865e12..fd1677cbf 100644
--- a/backend/device_registry/tests/test_api.py
+++ b/backend/device_registry/tests/test_api.py
@@ -868,7 +868,7 @@ def test_ping_writes_packages(self):
             'deb_packages_hash': 'abcdef'
         })
         self.device.refresh_from_db()
-        self.assertListEqual(self.device.deb_packages, packages)
+        self.assertListEqual(self.device.deb_packages['packages'], packages)
 
 
 class DeviceEnrollView(APITestCase):

From a0b1a205825d9225127484df8615350b433fce65 Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Mon, 2 Sep 2019 23:09:26 +0600
Subject: [PATCH 07/17] Display N/A for Insecure Services if no deb packages.

---
 .../templates/device_info_security.html            |  4 +++-
 backend/device_registry/tests/test_all.py          | 14 ++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/backend/device_registry/templates/device_info_security.html b/backend/device_registry/templates/device_info_security.html
index ad957d5eb..8710d152d 100644
--- a/backend/device_registry/templates/device_info_security.html
+++ b/backend/device_registry/templates/device_info_security.html
@@ -86,8 +86,10 @@ <h4 class="tab-title">Security</h4>
                       <li>{{ service_name }}</p>
                     {% endfor %}
                     </ul>
-                    {% else %}
+                    {% elif services is not None %}
                     No insecure services detected.
+                    {% else %}
+                    N/A
                     {% endif %}
                   {% endwith %}
                   </td>
diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py
index c0c01ac55..404222f36 100644
--- a/backend/device_registry/tests/test_all.py
+++ b/backend/device_registry/tests/test_all.py
@@ -536,6 +536,20 @@ def test_logins(self):
     def test_insecure_services(self):
         self.client.login(username='test', password='123')
         url = reverse('device-detail-security', kwargs={'pk': self.device.pk})
+        
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertNotContains(response, '>telnetd<')
+        self.assertNotContains(response, '>fingerd<')
+        self.assertNotContains(response, 'No insecure services detected')
+
+        self.device.deb_packages = {
+            'packages': [
+                {'name': 'python2', 'version': 'VERSION'},
+                {'name': 'python3', 'version': 'VERSION'}
+            ]
+        }
+        self.device.save()
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         self.assertNotContains(response, '>telnetd<')

From adb226382f332aee29f1079a84021970e133897e Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Tue, 3 Sep 2019 19:14:20 +0600
Subject: [PATCH 08/17] Make ping response flat again.

---
 backend/device_registry/api_views.py          |  8 ++---
 .../templates/device_info_security.html       |  2 +-
 backend/device_registry/tests/test_api.py     | 30 +++++++------------
 3 files changed, 15 insertions(+), 25 deletions(-)

diff --git a/backend/device_registry/api_views.py b/backend/device_registry/api_views.py
index a9279b19a..267ba2973 100644
--- a/backend/device_registry/api_views.py
+++ b/backend/device_registry/api_views.py
@@ -59,11 +59,9 @@ def get(self, request, *args, **kwargs):
         block_networks = portscan_object.block_networks.copy()
         block_networks.extend(settings.SPAM_NETWORKS)
         return Response({
-            'firewall': {
-                'policy': firewallstate_object.policy_string,
-                firewallstate_object.ports_field_name: portscan_object.block_ports,
-                'block_networks': block_networks
-            },
+            'policy': firewallstate_object.policy_string,
+            firewallstate_object.ports_field_name: portscan_object.block_ports,
+            'block_networks': block_networks,
             'deb_packages_hash': device.deb_packages.get('hash')
         })
 
diff --git a/backend/device_registry/templates/device_info_security.html b/backend/device_registry/templates/device_info_security.html
index 8710d152d..4c4c9a432 100644
--- a/backend/device_registry/templates/device_info_security.html
+++ b/backend/device_registry/templates/device_info_security.html
@@ -83,7 +83,7 @@ <h4 class="tab-title">Security</h4>
                     {% if services %}
                     <ul>
                     {% for service_name in services %}
-                      <li>{{ service_name }}</p>
+                      <li>{{ service_name }}</li>
                     {% endfor %}
                     </ul>
                     {% elif services is not None %}
diff --git a/backend/device_registry/tests/test_api.py b/backend/device_registry/tests/test_api.py
index fd1677cbf..92bfbac93 100644
--- a/backend/device_registry/tests/test_api.py
+++ b/backend/device_registry/tests/test_api.py
@@ -718,10 +718,8 @@ def test_ping_get_success(self):
         response = self.client.get(self.url, **self.headers)
         self.assertEqual(response.status_code, 200)
         self.assertDictEqual(response.data, {
-            'firewall': {
-                'policy': self.device.firewallstate.policy_string,
-                'block_ports': [], 'block_networks': settings.SPAM_NETWORKS
-            },
+            'policy': self.device.firewallstate.policy_string,
+            'block_ports': [], 'block_networks': settings.SPAM_NETWORKS,
             'deb_packages_hash': None
         })
 
@@ -730,11 +728,9 @@ def test_pong_data(self):
         response = self.client.get(self.url, **self.headers)
         self.assertEqual(response.status_code, 200)
         self.assertDictEqual(response.data, {
-            'firewall': {
-                'block_ports': [], 
-                'block_networks': settings.SPAM_NETWORKS,
-                'policy': self.device.firewallstate.policy_string
-            },
+            'block_ports': [],
+            'block_networks': settings.SPAM_NETWORKS,
+            'policy': self.device.firewallstate.policy_string,
             'deb_packages_hash': None
         })
         # 2nd request
@@ -747,11 +743,9 @@ def test_pong_data(self):
         response = self.client.get(self.url, **self.headers)
         self.assertEqual(response.status_code, 200)
         self.assertDictEqual(response.data, {
-            'firewall': {
-                'policy': self.device.firewallstate.policy_string,
-                'block_ports': [['192.168.1.178', 'tcp', 22, False]],
-                'block_networks': [['192.168.1.177', False]] + settings.SPAM_NETWORKS
-            },
+            'policy': self.device.firewallstate.policy_string,
+            'block_ports': [['192.168.1.178', 'tcp', 22, False]],
+            'block_networks': [['192.168.1.177', False]] + settings.SPAM_NETWORKS,
             'deb_packages_hash': None
         })
 
@@ -860,11 +854,9 @@ def test_ping_writes_packages(self):
         response = self.client.get(self.url, **self.headers)
         self.assertEqual(response.status_code, 200)
         self.assertDictEqual(response.data, {
-            'firewall': {
-                'block_ports': [],
-                'block_networks': settings.SPAM_NETWORKS,
-                'policy': self.device.firewallstate.policy_string
-            },
+            'block_ports': [],
+            'block_networks': settings.SPAM_NETWORKS,
+            'policy': self.device.firewallstate.policy_string,
             'deb_packages_hash': 'abcdef'
         })
         self.device.refresh_from_db()

From e08e53549ccf6b0bec89d45f3648af886f58586a Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Thu, 5 Sep 2019 16:14:16 +0600
Subject: [PATCH 09/17] Make deb_packages an m2m relation.

---
 backend/backend/settings/dev.py               |  3 +-
 .../migrations/0057_auto_20190904_0750.py     | 37 +++++++++++++++++++
 backend/device_registry/models.py             | 22 +++++++++--
 .../templates/device_info_security.html       |  4 +-
 4 files changed, 59 insertions(+), 7 deletions(-)
 create mode 100644 backend/device_registry/migrations/0057_auto_20190904_0750.py

diff --git a/backend/backend/settings/dev.py b/backend/backend/settings/dev.py
index 78fa67a90..c16edd5bb 100644
--- a/backend/backend/settings/dev.py
+++ b/backend/backend/settings/dev.py
@@ -22,6 +22,7 @@
     'django_extensions'
 ]
 
-IS_DEV = True
+IS_MTLS_API=IS_API=IS_DASH=IS_DEV = True
+ALLOWED_HOSTS += ['*']
 
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
diff --git a/backend/device_registry/migrations/0057_auto_20190904_0750.py b/backend/device_registry/migrations/0057_auto_20190904_0750.py
new file mode 100644
index 000000000..fff919970
--- /dev/null
+++ b/backend/device_registry/migrations/0057_auto_20190904_0750.py
@@ -0,0 +1,37 @@
+# Generated by Django 2.1.10 on 2019-09-04 07:50
+
+import device_registry.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('device_registry', '0056_device_deb_packages'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DebPackage',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=128)),
+                ('version', models.CharField(max_length=128)),
+                ('distro', models.CharField(choices=[(device_registry.models.Distro('debian'), 'debian'), (device_registry.models.Distro('raspbian'), 'raspbian'), (device_registry.models.Distro('ubuntu'), 'ubuntu')], max_length=128)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='device',
+            name='deb_packages_hash',
+            field=models.CharField(blank=True, max_length=16, null=True),
+        ),
+        migrations.RemoveField(
+            model_name='device',
+            name='deb_packages',
+        ),
+        migrations.AddField(
+            model_name='device',
+            name='deb_packages',
+            field=models.ManyToManyField(to='device_registry.DebPackage'),
+        ),
+    ]
diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py
index a5cf4f51d..27959660e 100644
--- a/backend/device_registry/models.py
+++ b/backend/device_registry/models.py
@@ -1,3 +1,4 @@
+from enum import Enum
 import datetime
 from statistics import mean
 import json
@@ -39,6 +40,18 @@ def from_db_value(self, value, expression, connection, context):
         return value
 
 
+class DebPackage(models.Model):
+    class Distro(Enum):
+        DEBIAN = 'debian'
+        RASPBIAN = 'raspbian'
+        UBUNTU = 'ubuntu'
+
+    name = models.CharField(max_length=128, null=False, blank=False)
+    version = models.CharField(max_length=128, null=False, blank=False)
+    distro = models.CharField(max_length=128, null=False, blank=False,
+                              choices=[(tag, tag.value) for tag in Distro])
+
+
 class Device(models.Model):
     device_id = models.CharField(
         max_length=128,
@@ -65,7 +78,8 @@ class Device(models.Model):
     agent_version = models.CharField(max_length=36, blank=True, null=True)
     tags = tagulous.models.TagField(to=Tag, blank=True)
     trust_score = models.FloatField(null=True)
-    deb_packages = JSONField(blank=True, default=dict)
+    deb_packages = models.ManyToManyField(DebPackage)
+    deb_packages_hash = models.CharField(max_length=16, blank=True, null=True)
 
     @property
     def certificate_expired(self):
@@ -85,9 +99,9 @@ def certificate_expired(self):
     ]
     @property
     def insecure_services(self):
-        if 'packages' in self.deb_packages:
-            packages = set([p['name'] for p in self.deb_packages['packages']])
-            return set(self.INSECURE_SERVICES) & packages
+        if not self.deb_packages_hash:
+            return None
+        return self.deb_packages.filter(name__in=self.INSECURE_SERVICES)
 
     def get_name(self):
         if self.name:
diff --git a/backend/device_registry/templates/device_info_security.html b/backend/device_registry/templates/device_info_security.html
index 4c4c9a432..6f5b2f2b7 100644
--- a/backend/device_registry/templates/device_info_security.html
+++ b/backend/device_registry/templates/device_info_security.html
@@ -82,8 +82,8 @@ <h4 class="tab-title">Security</h4>
                   {% with object.insecure_services as services %}
                     {% if services %}
                     <ul>
-                    {% for service_name in services %}
-                      <li>{{ service_name }}</li>
+                    {% for service in services %}
+                      <li>{{ service.name }}</li>
                     {% endfor %}
                     </ul>
                     {% elif services is not None %}

From 8444970a5aba3651b9b758383c6c560cbd50f3a1 Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Thu, 5 Sep 2019 18:21:47 +0600
Subject: [PATCH 10/17] Fix length of deb_packages_hash. Implement deb_packages
 as an m2m field.

---
 backend/device_registry/api_views.py          |  8 ++++--
 .../migrations/0057_auto_20190904_0750.py     |  5 +++-
 .../migrations/0058_auto_20190905_1152.py     | 18 ++++++++++++
 backend/device_registry/models.py             | 28 ++++++++++++++++++-
 4 files changed, 54 insertions(+), 5 deletions(-)
 create mode 100644 backend/device_registry/migrations/0058_auto_20190905_1152.py

diff --git a/backend/device_registry/api_views.py b/backend/device_registry/api_views.py
index 267ba2973..3dfc4c0f1 100644
--- a/backend/device_registry/api_views.py
+++ b/backend/device_registry/api_views.py
@@ -62,7 +62,7 @@ def get(self, request, *args, **kwargs):
             'policy': firewallstate_object.policy_string,
             firewallstate_object.ports_field_name: portscan_object.block_ports,
             'block_networks': block_networks,
-            'deb_packages_hash': device.deb_packages.get('hash')
+            'deb_packages_hash': device.deb_packages_hash
         })
 
     def post(self, request, *args, **kwargs):
@@ -71,7 +71,9 @@ def post(self, request, *args, **kwargs):
         device.last_ping = timezone.now()
         device.agent_version = data.get('agent_version')
         if 'deb_packages' in data:
-            device.deb_packages = data['deb_packages']
+            deb_packages = data['deb_packages']
+            device.deb_packages_hash = deb_packages.get('hash')
+            device.set_deb_packages(deb_packages.get('packages'))
 
         device_info_object, _ = DeviceInfo.objects.get_or_create(device=device)
         device_info_object.device__last_ping = timezone.now()
@@ -107,7 +109,7 @@ def post(self, request, *args, **kwargs):
         firewall_state.rules = firewall_rules
         firewall_state.save()
 
-        device.save(update_fields=['last_ping', 'agent_version', 'deb_packages', 'trust_score'])
+        device.save(update_fields=['last_ping', 'agent_version', 'deb_packages_hash', 'trust_score'])
 
         if datastore_client:
             task_key = datastore_client.key('Ping')
diff --git a/backend/device_registry/migrations/0057_auto_20190904_0750.py b/backend/device_registry/migrations/0057_auto_20190904_0750.py
index fff919970..fa307b03d 100644
--- a/backend/device_registry/migrations/0057_auto_20190904_0750.py
+++ b/backend/device_registry/migrations/0057_auto_20190904_0750.py
@@ -17,7 +17,10 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('name', models.CharField(max_length=128)),
                 ('version', models.CharField(max_length=128)),
-                ('distro', models.CharField(choices=[(device_registry.models.Distro('debian'), 'debian'), (device_registry.models.Distro('raspbian'), 'raspbian'), (device_registry.models.Distro('ubuntu'), 'ubuntu')], max_length=128)),
+                ('distro', models.CharField(choices=[(device_registry.models.DebPackage.Distro('debian'), 'debian'),
+                                                     (device_registry.models.DebPackage.Distro('raspbian'), 'raspbian'),
+                                                     (device_registry.models.DebPackage.Distro('ubuntu'), 'ubuntu')],
+                                            max_length=128)),
             ],
         ),
         migrations.AddField(
diff --git a/backend/device_registry/migrations/0058_auto_20190905_1152.py b/backend/device_registry/migrations/0058_auto_20190905_1152.py
new file mode 100644
index 000000000..1104fac63
--- /dev/null
+++ b/backend/device_registry/migrations/0058_auto_20190905_1152.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.1.10 on 2019-09-05 11:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('device_registry', '0057_auto_20190904_0750'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='device',
+            name='deb_packages_hash',
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+    ]
diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py
index 27959660e..060265098 100644
--- a/backend/device_registry/models.py
+++ b/backend/device_registry/models.py
@@ -79,7 +79,7 @@ class Device(models.Model):
     tags = tagulous.models.TagField(to=Tag, blank=True)
     trust_score = models.FloatField(null=True)
     deb_packages = models.ManyToManyField(DebPackage)
-    deb_packages_hash = models.CharField(max_length=16, blank=True, null=True)
+    deb_packages_hash = models.CharField(max_length=32, blank=True, null=True)
 
     @property
     def certificate_expired(self):
@@ -103,6 +103,32 @@ def insecure_services(self):
             return None
         return self.deb_packages.filter(name__in=self.INSECURE_SERVICES)
 
+    def set_deb_packages(self, packages):
+        packages_set = set((p['name'], p['version']) for p in packages)
+
+        # Find which packages we already have in db.
+        existing_packages = DebPackage.objects.filter(name__in=(p[0] for p in packages_set)).intersection(
+            DebPackage.objects.filter(version__in=(p[1] for p in packages_set)))
+        existing_packages_set = set((p.name, p.version) for p in existing_packages)
+
+        # Find the difference between the incoming package list and what we already have in db.
+        # Insert the missing packages.
+        extra_packages = packages_set.difference(existing_packages_set)
+        DebPackage.objects.bulk_create(DebPackage(name=p[0], version=p[1]) for p in extra_packages)
+
+        # Since bulk_create doesn't fetch created ids we need to do this ourselves.
+        extra_packages = DebPackage.objects.filter(name__in=(p[0] for p in extra_packages)).intersection(
+            DebPackage.objects.filter(version__in=(p[1] for p in extra_packages)))
+
+        # Re-create the m2m relation deb_packages in a bulk.
+        # The list of all packages is a union of existing_packages (which had existed in the db already) and
+        # extra_packages (which we've just created).
+        # FIXME: use new ignore_conflicts arg to bulk_create in Django 2.2.
+        Device.deb_packages.through.objects.all().delete()
+        Device.deb_packages.through.objects.bulk_create(
+            (Device.deb_packages.through(device_id=self.pk, debpackage_id=p.pk) for p in existing_packages|extra_packages)
+        )
+
     def get_name(self):
         if self.name:
             return self.name

From 2393f8f5bf5b7d5277b156cb61d83ae205ddeb06 Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Fri, 6 Sep 2019 15:08:14 +0600
Subject: [PATCH 11/17] Update tests to work with m2m deb_packages.

---
 backend/device_registry/tests/test_all.py | 22 ++++++++++------------
 backend/device_registry/tests/test_api.py |  5 +++--
 2 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py
index 404222f36..e13460337 100644
--- a/backend/device_registry/tests/test_all.py
+++ b/backend/device_registry/tests/test_all.py
@@ -543,12 +543,11 @@ def test_insecure_services(self):
         self.assertNotContains(response, '>fingerd<')
         self.assertNotContains(response, 'No insecure services detected')
 
-        self.device.deb_packages = {
-            'packages': [
-                {'name': 'python2', 'version': 'VERSION'},
-                {'name': 'python3', 'version': 'VERSION'}
-            ]
-        }
+        self.device.set_deb_packages([
+            {'name': 'python2', 'version': 'VERSION'},
+            {'name': 'python3', 'version': 'VERSION'}
+        ])
+        self.device.deb_packages_hash = 'abcdef'
         self.device.save()
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
@@ -556,17 +555,16 @@ def test_insecure_services(self):
         self.assertNotContains(response, '>fingerd<')
         self.assertContains(response, 'No insecure services detected')
 
-        self.device.deb_packages = {
-            'packages': [
-                {'name': 'telnetd', 'version': 'VERSION'},
-                {'name': 'fingerd', 'version': 'VERSION'}
-            ]
-        }
+        self.device.set_deb_packages([
+            {'name': 'telnetd', 'version': 'VERSION'},
+            {'name': 'fingerd', 'version': 'VERSION'}
+        ])
         self.device.save()
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         self.assertContains(response, '>telnetd<')
         self.assertContains(response, '>fingerd<')
+        self.assertNotContains(response, 'No insecure services detected')
 
 
 class PairingKeysView(TestCase):
diff --git a/backend/device_registry/tests/test_api.py b/backend/device_registry/tests/test_api.py
index 92bfbac93..3bfab9acc 100644
--- a/backend/device_registry/tests/test_api.py
+++ b/backend/device_registry/tests/test_api.py
@@ -17,7 +17,7 @@
 from rest_framework.exceptions import ErrorDetail
 from rest_framework.authtoken.models import Token
 
-from device_registry.models import Credential, Device, DeviceInfo, Tag, FirewallState, PortScan, PairingKey
+from device_registry.models import Credential, Device, DeviceInfo, Tag, FirewallState, PortScan, PairingKey, DebPackage
 from device_registry.serializers import DeviceListSerializer
 
 
@@ -860,7 +860,8 @@ def test_ping_writes_packages(self):
             'deb_packages_hash': 'abcdef'
         })
         self.device.refresh_from_db()
-        self.assertListEqual(self.device.deb_packages['packages'], packages)
+        self.assertQuerysetEqual(self.device.deb_packages.all(), packages,
+                                 transform=lambda p: {'name': p.name, 'version': p.version})
 
 
 class DeviceEnrollView(APITestCase):

From 4282e2730eb28a91d9d09fb172145f8c8c175c7f Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Sun, 8 Sep 2019 10:30:43 +0600
Subject: [PATCH 12/17] Remove dev code in dev.py. Remove unused import in
 test_api. Don't allow missing fields in deb_packages inside ping message. Add
 unique constraint to DebPackage model.

---
 backend/backend/settings/dev.py           |  3 +--
 backend/device_registry/api_views.py      |  4 ++--
 backend/device_registry/models.py         | 16 ++++++++++++----
 backend/device_registry/tests/test_api.py |  2 +-
 4 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/backend/backend/settings/dev.py b/backend/backend/settings/dev.py
index c16edd5bb..78fa67a90 100644
--- a/backend/backend/settings/dev.py
+++ b/backend/backend/settings/dev.py
@@ -22,7 +22,6 @@
     'django_extensions'
 ]
 
-IS_MTLS_API=IS_API=IS_DASH=IS_DEV = True
-ALLOWED_HOSTS += ['*']
+IS_DEV = True
 
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
diff --git a/backend/device_registry/api_views.py b/backend/device_registry/api_views.py
index 3dfc4c0f1..5288ba40b 100644
--- a/backend/device_registry/api_views.py
+++ b/backend/device_registry/api_views.py
@@ -72,8 +72,8 @@ def post(self, request, *args, **kwargs):
         device.agent_version = data.get('agent_version')
         if 'deb_packages' in data:
             deb_packages = data['deb_packages']
-            device.deb_packages_hash = deb_packages.get('hash')
-            device.set_deb_packages(deb_packages.get('packages'))
+            device.deb_packages_hash = deb_packages['hash']
+            device.set_deb_packages(deb_packages['packages'])
 
         device_info_object, _ = DeviceInfo.objects.get_or_create(device=device)
         device_info_object.device__last_ping = timezone.now()
diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py
index 060265098..3baf3cd03 100644
--- a/backend/device_registry/models.py
+++ b/backend/device_registry/models.py
@@ -46,10 +46,18 @@ class Distro(Enum):
         RASPBIAN = 'raspbian'
         UBUNTU = 'ubuntu'
 
-    name = models.CharField(max_length=128, null=False, blank=False)
-    version = models.CharField(max_length=128, null=False, blank=False)
-    distro = models.CharField(max_length=128, null=False, blank=False,
-                              choices=[(tag, tag.value) for tag in Distro])
+    class Arch(Enum):
+        i386 = 'i386'
+        amd64 = 'amd64'
+        armhf = 'armhf'
+
+    name = models.CharField(max_length=128)
+    version = models.CharField(max_length=128)
+    distro = models.CharField(max_length=128, choices=[(tag, tag.value) for tag in Distro])
+    arch = models.CharField(max_length=16, choices=[(tag, tag.value) for tag in Distro])
+
+    class Meta:
+        unique_together = ['name', 'version', 'distro']
 
 
 class Device(models.Model):
diff --git a/backend/device_registry/tests/test_api.py b/backend/device_registry/tests/test_api.py
index 3bfab9acc..10110346a 100644
--- a/backend/device_registry/tests/test_api.py
+++ b/backend/device_registry/tests/test_api.py
@@ -17,7 +17,7 @@
 from rest_framework.exceptions import ErrorDetail
 from rest_framework.authtoken.models import Token
 
-from device_registry.models import Credential, Device, DeviceInfo, Tag, FirewallState, PortScan, PairingKey, DebPackage
+from device_registry.models import Credential, Device, DeviceInfo, Tag, FirewallState, PortScan, PairingKey
 from device_registry.serializers import DeviceListSerializer
 
 

From e6e826f9b4d1123f4c6579cd6eb92626101d0606 Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Sun, 8 Sep 2019 11:06:53 +0600
Subject: [PATCH 13/17] Add missing migration.

---
 .../migrations/0059_auto_20190908_0452.py     | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 backend/device_registry/migrations/0059_auto_20190908_0452.py

diff --git a/backend/device_registry/migrations/0059_auto_20190908_0452.py b/backend/device_registry/migrations/0059_auto_20190908_0452.py
new file mode 100644
index 000000000..ea9e0f418
--- /dev/null
+++ b/backend/device_registry/migrations/0059_auto_20190908_0452.py
@@ -0,0 +1,24 @@
+# Generated by Django 2.1.10 on 2019-09-08 04:52
+
+import device_registry.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('device_registry', '0058_auto_20190905_1152'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='debpackage',
+            name='arch',
+            field=models.CharField(choices=[(device_registry.models.DebPackage.Distro('debian'), 'debian'), (device_registry.models.DebPackage.Distro('raspbian'), 'raspbian'), (device_registry.models.DebPackage.Distro('ubuntu'), 'ubuntu')], default='i386', max_length=16),
+            preserve_default=False,
+        ),
+        migrations.AlterUniqueTogether(
+            name='debpackage',
+            unique_together={('name', 'version', 'distro')},
+        ),
+    ]

From 747ea8afa70f25fb82616aa1e0024c3363bd2bae Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Mon, 9 Sep 2019 17:27:46 +0600
Subject: [PATCH 14/17] Remove DebPackage.distro, add unique constraint and
 Arch.all. Merge migrations.

---
 ...904_0750.py => 0056_auto_20190909_1128.py} | 17 +++++-------
 .../migrations/0056_device_deb_packages.py    | 19 --------------
 .../migrations/0058_auto_20190905_1152.py     | 18 -------------
 .../migrations/0059_auto_20190908_0452.py     | 24 -----------------
 backend/device_registry/models.py             | 26 ++++++++++---------
 backend/device_registry/tests/test_all.py     |  8 +++---
 backend/device_registry/tests/test_api.py     |  4 +--
 7 files changed, 27 insertions(+), 89 deletions(-)
 rename backend/device_registry/migrations/{0057_auto_20190904_0750.py => 0056_auto_20190909_1128.py} (54%)
 delete mode 100644 backend/device_registry/migrations/0056_device_deb_packages.py
 delete mode 100644 backend/device_registry/migrations/0058_auto_20190905_1152.py
 delete mode 100644 backend/device_registry/migrations/0059_auto_20190908_0452.py

diff --git a/backend/device_registry/migrations/0057_auto_20190904_0750.py b/backend/device_registry/migrations/0056_auto_20190909_1128.py
similarity index 54%
rename from backend/device_registry/migrations/0057_auto_20190904_0750.py
rename to backend/device_registry/migrations/0056_auto_20190909_1128.py
index fa307b03d..19b69c4af 100644
--- a/backend/device_registry/migrations/0057_auto_20190904_0750.py
+++ b/backend/device_registry/migrations/0056_auto_20190909_1128.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.1.10 on 2019-09-04 07:50
+# Generated by Django 2.1.10 on 2019-09-09 11:28
 
 import device_registry.models
 from django.db import migrations, models
@@ -7,7 +7,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('device_registry', '0056_device_deb_packages'),
+        ('device_registry', '0055_remove_deviceinfo_trust_score'),
     ]
 
     operations = [
@@ -17,20 +17,17 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('name', models.CharField(max_length=128)),
                 ('version', models.CharField(max_length=128)),
-                ('distro', models.CharField(choices=[(device_registry.models.DebPackage.Distro('debian'), 'debian'),
-                                                     (device_registry.models.DebPackage.Distro('raspbian'), 'raspbian'),
-                                                     (device_registry.models.DebPackage.Distro('ubuntu'), 'ubuntu')],
-                                            max_length=128)),
+                ('arch', models.CharField(choices=[(device_registry.models.DebPackage.Arch('i386'), 'i386'), (device_registry.models.DebPackage.Arch('amd64'), 'amd64'), (device_registry.models.DebPackage.Arch('armhf'), 'armhf'), (device_registry.models.DebPackage.Arch('all'), 'all')], max_length=16)),
             ],
         ),
         migrations.AddField(
             model_name='device',
             name='deb_packages_hash',
-            field=models.CharField(blank=True, max_length=16, null=True),
+            field=models.CharField(blank=True, max_length=32, null=True),
         ),
-        migrations.RemoveField(
-            model_name='device',
-            name='deb_packages',
+        migrations.AlterUniqueTogether(
+            name='debpackage',
+            unique_together={('name', 'version', 'arch')},
         ),
         migrations.AddField(
             model_name='device',
diff --git a/backend/device_registry/migrations/0056_device_deb_packages.py b/backend/device_registry/migrations/0056_device_deb_packages.py
deleted file mode 100644
index 32a2b9a43..000000000
--- a/backend/device_registry/migrations/0056_device_deb_packages.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Generated by Django 2.1.10 on 2019-08-30 10:48
-
-import django.contrib.postgres.fields.jsonb
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('device_registry', '0055_remove_deviceinfo_trust_score'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='device',
-            name='deb_packages',
-            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict),
-        ),
-    ]
diff --git a/backend/device_registry/migrations/0058_auto_20190905_1152.py b/backend/device_registry/migrations/0058_auto_20190905_1152.py
deleted file mode 100644
index 1104fac63..000000000
--- a/backend/device_registry/migrations/0058_auto_20190905_1152.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 2.1.10 on 2019-09-05 11:52
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('device_registry', '0057_auto_20190904_0750'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='device',
-            name='deb_packages_hash',
-            field=models.CharField(blank=True, max_length=32, null=True),
-        ),
-    ]
diff --git a/backend/device_registry/migrations/0059_auto_20190908_0452.py b/backend/device_registry/migrations/0059_auto_20190908_0452.py
deleted file mode 100644
index ea9e0f418..000000000
--- a/backend/device_registry/migrations/0059_auto_20190908_0452.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Generated by Django 2.1.10 on 2019-09-08 04:52
-
-import device_registry.models
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('device_registry', '0058_auto_20190905_1152'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='debpackage',
-            name='arch',
-            field=models.CharField(choices=[(device_registry.models.DebPackage.Distro('debian'), 'debian'), (device_registry.models.DebPackage.Distro('raspbian'), 'raspbian'), (device_registry.models.DebPackage.Distro('ubuntu'), 'ubuntu')], default='i386', max_length=16),
-            preserve_default=False,
-        ),
-        migrations.AlterUniqueTogether(
-            name='debpackage',
-            unique_together={('name', 'version', 'distro')},
-        ),
-    ]
diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py
index 3baf3cd03..8b81e80fe 100644
--- a/backend/device_registry/models.py
+++ b/backend/device_registry/models.py
@@ -48,16 +48,16 @@ class Distro(Enum):
 
     class Arch(Enum):
         i386 = 'i386'
-        amd64 = 'amd64'
-        armhf = 'armhf'
+        AMD64 = 'amd64'
+        ARMHF = 'armhf'
+        ALL = 'all'
 
     name = models.CharField(max_length=128)
     version = models.CharField(max_length=128)
-    distro = models.CharField(max_length=128, choices=[(tag, tag.value) for tag in Distro])
-    arch = models.CharField(max_length=16, choices=[(tag, tag.value) for tag in Distro])
+    arch = models.CharField(max_length=16, choices=[(tag, tag.value) for tag in Arch])
 
     class Meta:
-        unique_together = ['name', 'version', 'distro']
+        unique_together = ['name', 'version', 'arch']
 
 
 class Device(models.Model):
@@ -112,21 +112,23 @@ def insecure_services(self):
         return self.deb_packages.filter(name__in=self.INSECURE_SERVICES)
 
     def set_deb_packages(self, packages):
-        packages_set = set((p['name'], p['version']) for p in packages)
+        packages_set = set((p['name'], p['version'], p['arch']) for p in packages)
 
         # Find which packages we already have in db.
-        existing_packages = DebPackage.objects.filter(name__in=(p[0] for p in packages_set)).intersection(
-            DebPackage.objects.filter(version__in=(p[1] for p in packages_set)))
-        existing_packages_set = set((p.name, p.version) for p in existing_packages)
+        existing_packages = DebPackage.objects.filter(name__in=(p[0] for p in packages_set))\
+            .filter(version__in=(p[1] for p in packages_set))\
+            .filter(arch__in=(p[2] for p in packages_set))
+        existing_packages_set = set((p.name, p.version, p.arch) for p in existing_packages)
 
         # Find the difference between the incoming package list and what we already have in db.
         # Insert the missing packages.
         extra_packages = packages_set.difference(existing_packages_set)
-        DebPackage.objects.bulk_create(DebPackage(name=p[0], version=p[1]) for p in extra_packages)
+        DebPackage.objects.bulk_create(DebPackage(name=p[0], version=p[1], arch=p[2]) for p in extra_packages)
 
         # Since bulk_create doesn't fetch created ids we need to do this ourselves.
-        extra_packages = DebPackage.objects.filter(name__in=(p[0] for p in extra_packages)).intersection(
-            DebPackage.objects.filter(version__in=(p[1] for p in extra_packages)))
+        extra_packages = DebPackage.objects.filter(name__in=(p[0] for p in extra_packages))\
+            .filter(version__in=(p[1] for p in extra_packages))\
+            .filter(arch__in=(p[2] for p in extra_packages))
 
         # Re-create the m2m relation deb_packages in a bulk.
         # The list of all packages is a union of existing_packages (which had existed in the db already) and
diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py
index e13460337..97f075290 100644
--- a/backend/device_registry/tests/test_all.py
+++ b/backend/device_registry/tests/test_all.py
@@ -544,8 +544,8 @@ def test_insecure_services(self):
         self.assertNotContains(response, 'No insecure services detected')
 
         self.device.set_deb_packages([
-            {'name': 'python2', 'version': 'VERSION'},
-            {'name': 'python3', 'version': 'VERSION'}
+            {'name': 'python2', 'version': 'VERSION', 'arch': 'i386'},
+            {'name': 'python3', 'version': 'VERSION', 'arch': 'i386'}
         ])
         self.device.deb_packages_hash = 'abcdef'
         self.device.save()
@@ -556,8 +556,8 @@ def test_insecure_services(self):
         self.assertContains(response, 'No insecure services detected')
 
         self.device.set_deb_packages([
-            {'name': 'telnetd', 'version': 'VERSION'},
-            {'name': 'fingerd', 'version': 'VERSION'}
+            {'name': 'telnetd', 'version': 'VERSION', 'arch': 'i386'},
+            {'name': 'fingerd', 'version': 'VERSION', 'arch': 'i386'}
         ])
         self.device.save()
         response = self.client.get(url)
diff --git a/backend/device_registry/tests/test_api.py b/backend/device_registry/tests/test_api.py
index 10110346a..48f678ec3 100644
--- a/backend/device_registry/tests/test_api.py
+++ b/backend/device_registry/tests/test_api.py
@@ -844,7 +844,7 @@ def test_ping_writes_trust_score(self):
         self.assertGreater(self.device.trust_score, 0.42)
 
     def test_ping_writes_packages(self):
-        packages = [{'name': 'PACKAGE', 'version': 'VERSION'}]
+        packages = [{'name': 'PACKAGE', 'version': 'VERSION', 'arch': 'all'}]
         self.ping_payload['deb_packages'] = {
             'hash': 'abcdef',
             'packages': packages
@@ -861,7 +861,7 @@ def test_ping_writes_packages(self):
         })
         self.device.refresh_from_db()
         self.assertQuerysetEqual(self.device.deb_packages.all(), packages,
-                                 transform=lambda p: {'name': p.name, 'version': p.version})
+                                 transform=lambda p: {'name': p.name, 'version': p.version, 'arch': p.arch})
 
 
 class DeviceEnrollView(APITestCase):

From c264acd9f2dfb0f02cc67ca1f176571f833eb32c Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Mon, 9 Sep 2019 18:43:00 +0600
Subject: [PATCH 15/17] Don't allow blank in deb_packages_hash.

---
 .../device_registry/migrations/0056_auto_20190909_1128.py   | 2 +-
 backend/device_registry/models.py                           | 2 +-
 backend/device_registry/tests/test_api.py                   | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/backend/device_registry/migrations/0056_auto_20190909_1128.py b/backend/device_registry/migrations/0056_auto_20190909_1128.py
index 19b69c4af..64542024e 100644
--- a/backend/device_registry/migrations/0056_auto_20190909_1128.py
+++ b/backend/device_registry/migrations/0056_auto_20190909_1128.py
@@ -23,7 +23,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='device',
             name='deb_packages_hash',
-            field=models.CharField(blank=True, max_length=32, null=True),
+            field=models.CharField(blank=True, max_length=32),
         ),
         migrations.AlterUniqueTogether(
             name='debpackage',
diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py
index 8b81e80fe..eca67837e 100644
--- a/backend/device_registry/models.py
+++ b/backend/device_registry/models.py
@@ -87,7 +87,7 @@ class Device(models.Model):
     tags = tagulous.models.TagField(to=Tag, blank=True)
     trust_score = models.FloatField(null=True)
     deb_packages = models.ManyToManyField(DebPackage)
-    deb_packages_hash = models.CharField(max_length=32, blank=True, null=True)
+    deb_packages_hash = models.CharField(max_length=32, blank=True)
 
     @property
     def certificate_expired(self):
diff --git a/backend/device_registry/tests/test_api.py b/backend/device_registry/tests/test_api.py
index 48f678ec3..ac4c5f8e4 100644
--- a/backend/device_registry/tests/test_api.py
+++ b/backend/device_registry/tests/test_api.py
@@ -720,7 +720,7 @@ def test_ping_get_success(self):
         self.assertDictEqual(response.data, {
             'policy': self.device.firewallstate.policy_string,
             'block_ports': [], 'block_networks': settings.SPAM_NETWORKS,
-            'deb_packages_hash': None
+            'deb_packages_hash': ''
         })
 
     def test_pong_data(self):
@@ -731,7 +731,7 @@ def test_pong_data(self):
             'block_ports': [],
             'block_networks': settings.SPAM_NETWORKS,
             'policy': self.device.firewallstate.policy_string,
-            'deb_packages_hash': None
+            'deb_packages_hash': ''
         })
         # 2nd request
         self.device.portscan.block_ports = [['192.168.1.178', 'tcp', 22, False]]
@@ -746,7 +746,7 @@ def test_pong_data(self):
             'policy': self.device.firewallstate.policy_string,
             'block_ports': [['192.168.1.178', 'tcp', 22, False]],
             'block_networks': [['192.168.1.177', False]] + settings.SPAM_NETWORKS,
-            'deb_packages_hash': None
+            'deb_packages_hash': ''
         })
 
     def test_ping_creates_models(self):

From c76eece775b20c0e5c752991aa45eed596732b89 Mon Sep 17 00:00:00 2001
From: Artem Martynovich <artem.martynovich@gmail.com>
Date: Mon, 9 Sep 2019 18:50:20 +0600
Subject: [PATCH 16/17] Add docstrings to insecure_services() and
 set_deb_packages().

---
 backend/device_registry/models.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py
index eca67837e..120b455bd 100644
--- a/backend/device_registry/models.py
+++ b/backend/device_registry/models.py
@@ -107,11 +107,19 @@ def certificate_expired(self):
     ]
     @property
     def insecure_services(self):
+        """
+        Get a list of deb packages which are marked "insecure", i.e. their names are in INSECURE_SERVICES list.
+        :return: list of DebPackage or None if set_deb_packages() wasn't called before.
+        """
         if not self.deb_packages_hash:
             return None
         return self.deb_packages.filter(name__in=self.INSECURE_SERVICES)
 
     def set_deb_packages(self, packages):
+        """
+        Assign the list of installed deb packages to this device.
+        :param packages: list of dicts with the following values: 'name': str, 'version': str, 'arch': DebPackage.Arch.
+        """
         packages_set = set((p['name'], p['version'], p['arch']) for p in packages)
 
         # Find which packages we already have in db.

From 7d0c78aef3d097da835813e83c4760ebf2bf7968 Mon Sep 17 00:00:00 2001
From: Roman P <pythonpro@gmail.com>
Date: Wed, 11 Sep 2019 08:01:23 +0400
Subject: [PATCH 17/17] Optimize 'set_deb_packages'

---
 backend/device_registry/models.py         | 41 +++++++----------------
 backend/device_registry/tests/test_all.py | 14 ++++++--
 2 files changed, 24 insertions(+), 31 deletions(-)

diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py
index 120b455bd..4b3213fb6 100644
--- a/backend/device_registry/models.py
+++ b/backend/device_registry/models.py
@@ -5,10 +5,9 @@
 import uuid
 
 from django.conf import settings
-from django.db import models
+from django.db import models, transaction
 from django.utils import timezone
 from django.contrib.postgres.fields import JSONField
-from django.db import transaction
 
 import yaml
 import tagulous.models
@@ -105,6 +104,7 @@ def certificate_expired(self):
         'rsh-server',
         'rsh-redone-server'
     ]
+
     @property
     def insecure_services(self):
         """
@@ -120,32 +120,17 @@ def set_deb_packages(self, packages):
         Assign the list of installed deb packages to this device.
         :param packages: list of dicts with the following values: 'name': str, 'version': str, 'arch': DebPackage.Arch.
         """
-        packages_set = set((p['name'], p['version'], p['arch']) for p in packages)
-
-        # Find which packages we already have in db.
-        existing_packages = DebPackage.objects.filter(name__in=(p[0] for p in packages_set))\
-            .filter(version__in=(p[1] for p in packages_set))\
-            .filter(arch__in=(p[2] for p in packages_set))
-        existing_packages_set = set((p.name, p.version, p.arch) for p in existing_packages)
-
-        # Find the difference between the incoming package list and what we already have in db.
-        # Insert the missing packages.
-        extra_packages = packages_set.difference(existing_packages_set)
-        DebPackage.objects.bulk_create(DebPackage(name=p[0], version=p[1], arch=p[2]) for p in extra_packages)
-
-        # Since bulk_create doesn't fetch created ids we need to do this ourselves.
-        extra_packages = DebPackage.objects.filter(name__in=(p[0] for p in extra_packages))\
-            .filter(version__in=(p[1] for p in extra_packages))\
-            .filter(arch__in=(p[2] for p in extra_packages))
-
-        # Re-create the m2m relation deb_packages in a bulk.
-        # The list of all packages is a union of existing_packages (which had existed in the db already) and
-        # extra_packages (which we've just created).
-        # FIXME: use new ignore_conflicts arg to bulk_create in Django 2.2.
-        Device.deb_packages.through.objects.all().delete()
-        Device.deb_packages.through.objects.bulk_create(
-            (Device.deb_packages.through(device_id=self.pk, debpackage_id=p.pk) for p in existing_packages|extra_packages)
-        )
+        # Save new packages to DB.
+        DebPackage.objects.bulk_create([DebPackage(name=package['name'], version=package['version'],
+                                                   arch=package['arch']) for package in packages],
+                                       ignore_conflicts=True)
+        # Get packages qs.
+        q_objects = models.Q()
+        for package in packages:
+            q_objects.add(models.Q(name=package['name'], version=package['version'], arch=package['arch']), models.Q.OR)
+
+        # Set deb_packages.
+        self.deb_packages.set(DebPackage.objects.filter(q_objects).only('pk'))
 
     def get_name(self):
         if self.name:
diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py
index 97f075290..303ecd8d3 100644
--- a/backend/device_registry/tests/test_all.py
+++ b/backend/device_registry/tests/test_all.py
@@ -532,11 +532,11 @@ def test_logins(self):
         self.assertEqual(response.status_code, 200)
         self.assertContains(response, '<pre>pi:')
         self.assertContains(response, 'success: 1')
-    
+
     def test_insecure_services(self):
         self.client.login(username='test', password='123')
         url = reverse('device-detail-security', kwargs={'pk': self.device.pk})
-        
+
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         self.assertNotContains(response, '>telnetd<')
@@ -554,6 +554,10 @@ def test_insecure_services(self):
         self.assertNotContains(response, '>telnetd<')
         self.assertNotContains(response, '>fingerd<')
         self.assertContains(response, 'No insecure services detected')
+        self.assertListEqual(list(self.device.deb_packages.values('name', 'version', 'arch')), [
+            {'name': 'python2', 'version': 'VERSION', 'arch': 'i386'},
+            {'name': 'python3', 'version': 'VERSION', 'arch': 'i386'}
+        ])
 
         self.device.set_deb_packages([
             {'name': 'telnetd', 'version': 'VERSION', 'arch': 'i386'},
@@ -565,6 +569,10 @@ def test_insecure_services(self):
         self.assertContains(response, '>telnetd<')
         self.assertContains(response, '>fingerd<')
         self.assertNotContains(response, 'No insecure services detected')
+        self.assertListEqual(list(self.device.deb_packages.values('name', 'version', 'arch')), [
+            {'name': 'telnetd', 'version': 'VERSION', 'arch': 'i386'},
+            {'name': 'fingerd', 'version': 'VERSION', 'arch': 'i386'}
+        ])
 
 
 class PairingKeysView(TestCase):
@@ -605,7 +613,7 @@ def setUp(self):
             owner=self.user,
             certificate=TEST_CERT,
             name='First',
-            last_ping=timezone.now()-datetime.timedelta(days=1, hours=1)
+            last_ping=timezone.now() - datetime.timedelta(days=1, hours=1)
         )
         self.deviceinfo0 = DeviceInfo.objects.create(
             device=self.device0,