ElementsProject · cdecker · Nov 12, 2019 · Oct 26, 2019 · Oct 26, 2019 · Oct 27, 2019
diff --git a/Makefile b/Makefile
@@ -263,7 +263,7 @@ ifeq ($(PYTEST),)
 	exit 1
 else
 # Explicitly hand DEVELOPER and VALGRIND so you can override on make cmd line.
-	PYTHONPATH=`pwd`/contrib/pylightning:$$PYTHONPATH TEST_DEBUG=1 DEVELOPER=$(DEVELOPER) VALGRIND=$(VALGRIND) $(PYTEST) tests/ $(PYTEST_OPTS)
+	PYTHONPATH=`pwd`/contrib/pyln-client:`pwd`/contrib/pyln-testing:$$PYTHONPATH TEST_DEBUG=1 DEVELOPER=$(DEVELOPER) VALGRIND=$(VALGRIND) $(PYTEST) tests/ $(PYTEST_OPTS)
 endif
 
 # Keep includes in alpha order.
@@ -316,7 +316,7 @@ check-python:
 	@# W503: line break before binary operator
 	@flake8 --ignore=E501,E731,W503 --exclude=contrib/pylightning/lightning/__init__.py ${PYSRC}
 
-	PYTHONPATH=contrib/pylightning:$$PYTHONPATH $(PYTEST) contrib/pylightning/
+	PYTHONPATH=contrib/pyln-client:$$PYTHONPATH $(PYTEST) contrib/pyln-client/
 
 check-includes:
 	@tools/check-includes.sh

diff --git a/contrib/pylightning/lightning/__init__.py b/contrib/pylightning/lightning/__init__.py
@@ -1,2 +1 @@
-from .lightning import LightningRpc, RpcError, Millisatoshi, __version__
-from .plugin import Plugin, monkey_patch
+from pyln.client import LightningRpc, RpcError, Millisatoshi, __version__, Plugin, monkey_patch
diff --git a/contrib/pylightning/requirements.txt b/contrib/pylightning/requirements.txt
@@ -0,0 +1 @@
+pyln-client==0.7.3
@@ -1,4 +1,6 @@
-from lightning import LightningRpc, Plugin, RpcError, Millisatoshi, __version__, monkey_patch
+from .lightning import LightningRpc, RpcError, Millisatoshi, __version__
+from .plugin import Plugin, monkey_patch
+
 
 __all__ = [
     "LightningRpc",

@@ -5,7 +5,7 @@
 from decimal import Decimal
 from math import floor, log10
 
-__version__ = "0.0.7.4"
+__version__ = "0.7.3"
 
 
 class RpcError(ValueError):

@@ -1,6 +1,6 @@
 from collections import OrderedDict
 from enum import Enum
-from lightning import LightningRpc, Millisatoshi
+from .lightning import LightningRpc, Millisatoshi
 from threading import RLock
 
 import inspect

@@ -1 +0,0 @@
-pylightning==0.0.7.3

@@ -1,4 +1,4 @@
-from lightning import Millisatoshi
+from pyln.client import Millisatoshi
 
 
 def test_sum_radd():

@@ -1,5 +1,5 @@
-from lightning import Plugin
-from lightning.plugin import Request, Millisatoshi
+from pyln.client import Plugin
+from pyln.client.plugin import Request, Millisatoshi
 import itertools
 import pytest
 

@@ -1,4 +1,4 @@
-from lightning import Millisatoshi
+from pyln.client import Millisatoshi
 
 
 def test_to_approx_str():

@@ -0,0 +1,33 @@
+# pyln-testing: A library to write tests against c-lightning
+
+This library implements a number of utilities that help building tests for
+c-lightning nodes. In particular it provides a number of pytest fixtures that
+allow the management of a test network of a given topology and then execute a
+test scenarion.
+
+`pyln-testing` is used by c-lightning for its internal tests, and by the
+community plugin directory to exercise the plugins.
+
+## Installation
+
+`pyln-testing` is available on `pip`:
+
+```bash
+pip install pyln-testing
+```
+
+Alternatively you can also install the development version to get access to
+currently unreleased features by checking out the c-lightning source code and
+installing into your python3 environment:
+
+```bash
+git clone https://github.com/ElementsProject/lightning.git
+cd lightning/contrib/pyln-testing
+python3 setup.py develop
+```
+
+This will add links to the library into your environment so changing the
+checked out source code will also result in the environment picking up these
+changes. Notice however that unreleased versions may change API without
+warning, so test thoroughly with the released version.
+
@@ -0,0 +1 @@
+__version__ = "0.0.1"
@@ -0,0 +1,197 @@
+from ephemeral_port_reserve import reserve
+from glob import glob
+
+import logging
+import os
+import psycopg2
+import random
+import re
+import shutil
+import signal
+import sqlite3
+import string
+import subprocess
+import time
+
+
+class Sqlite3Db(object):
+    def __init__(self, path):
+        self.path = path
+
+    def get_dsn(self):
+        """SQLite3 doesn't provide a DSN, resulting in no CLI-option.
+        """
+        return None
+
+    def query(self, query):
+        orig = os.path.join(self.path)
+        copy = self.path + ".copy"
+        shutil.copyfile(orig, copy)
+        db = sqlite3.connect(copy)
+
+        db.row_factory = sqlite3.Row
+        c = db.cursor()
+        c.execute(query)
+        rows = c.fetchall()
+
+        result = []
+        for row in rows:
+            result.append(dict(zip(row.keys(), row)))
+
+        db.commit()
+        c.close()
+        db.close()
+        return result
+
+    def execute(self, query):
+        db = sqlite3.connect(self.path)
+        c = db.cursor()
+        c.execute(query)
+        db.commit()
+        c.close()
+        db.close()
+
+
+class PostgresDb(object):
+    def __init__(self, dbname, port):
+        self.dbname = dbname
+        self.port = port
+
+        self.conn = psycopg2.connect("dbname={dbname} user=postgres host=localhost port={port}".format(
+            dbname=dbname, port=port
+        ))
+        cur = self.conn.cursor()
+        cur.execute('SELECT 1')
+        cur.close()
+
+    def get_dsn(self):
+        return "postgres://postgres:password@localhost:{port}/{dbname}".format(
+            port=self.port, dbname=self.dbname
+        )
+
+    def query(self, query):
+        cur = self.conn.cursor()
+        cur.execute(query)
+
+        # Collect the results into a list of dicts.
+        res = []
+        for r in cur:
+            t = {}
+            # Zip the column definition with the value to get its name.
+            for c, v in zip(cur.description, r):
+                t[c.name] = v
+            res.append(t)
+        cur.close()
+        return res
+
+    def execute(self, query):
+        with self.conn, self.conn.cursor() as cur:
+            cur.execute(query)
+
+
+class SqliteDbProvider(object):
+    def __init__(self, directory):
+        self.directory = directory
+
+    def start(self):
+        pass
+
+    def get_db(self, node_directory, testname, node_id):
+        path = os.path.join(
+            node_directory,
+            'lightningd.sqlite3'
+        )
+        return Sqlite3Db(path)
+
+    def stop(self):
+        pass
+
+
+class PostgresDbProvider(object):
+    def __init__(self, directory):
+        self.directory = directory
+        self.port = None
+        self.proc = None
+        print("Starting PostgresDbProvider")
+
+    def locate_path(self):
+        prefix = '/usr/lib/postgresql/*'
+        matches = glob(prefix)
+
+        candidates = {}
+        for m in matches:
+            g = re.search(r'([0-9]+[\.0-9]*)', m)
+            if not g:
+                continue
+            candidates[float(g.group(1))] = m
+
+        if len(candidates) == 0:
+            raise ValueError("Could not find `postgres` and `initdb` binaries in {}. Is postgresql installed?".format(prefix))
+
+        # Now iterate in reverse order through matches
+        for k, v in sorted(candidates.items())[::-1]:
+            initdb = os.path.join(v, 'bin', 'initdb')
+            postgres = os.path.join(v, 'bin', 'postgres')
+            if os.path.isfile(initdb) and os.path.isfile(postgres):
+                logging.info("Found `postgres` and `initdb` in {}".format(os.path.join(v, 'bin')))
+                return initdb, postgres
+
+        raise ValueError("Could not find `postgres` and `initdb` in any of the possible paths: {}".format(candidates.values()))
+
+    def start(self):
+        passfile = os.path.join(self.directory, "pgpass.txt")
+        self.pgdir = os.path.join(self.directory, 'pgsql')
+        # Need to write a tiny file containing the password so `initdb` can pick it up
+        with open(passfile, 'w') as f:
+            f.write('cltest\n')
+
+        initdb, postgres = self.locate_path()
+        subprocess.check_call([
+            initdb,
+            '--pwfile={}'.format(passfile),
+            '--pgdata={}'.format(self.pgdir),
+            '--auth=trust',
+            '--username=postgres',
+        ])
+        self.port = reserve()
+        self.proc = subprocess.Popen([
+            postgres,
+            '-k', '/tmp/',  # So we don't use /var/lib/...
+            '-D', self.pgdir,
+            '-p', str(self.port),
+            '-F',
+            '-i',
+        ])
+        # Hacky but seems to work ok (might want to make the postgres proc a TailableProc as well if too flaky).
+        time.sleep(1)
+        self.conn = psycopg2.connect("dbname=template1 user=postgres host=localhost port={}".format(self.port))
+
+        # Required for CREATE DATABASE to work
+        self.conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+
+    def get_db(self, node_directory, testname, node_id):
+        # Random suffix to avoid collisions on repeated tests
+        nonce = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
+        dbname = "{}_{}_{}".format(testname, node_id, nonce)
+
+        cur = self.conn.cursor()
+        cur.execute("CREATE DATABASE {};".format(dbname))
+        cur.close()
+        db = PostgresDb(dbname, self.port)
+        return db
+
+    def stop(self):
+        # Send fast shutdown signal see [1] for details:
+        #
+        # SIGINT
+        #
+        # This is the Fast Shutdown mode. The server disallows new connections
+        # and sends all existing server processes SIGTERM, which will cause
+        # them to abort their current transactions and exit promptly. It then
+        # waits for all server processes to exit and finally shuts down. If
+        # the server is in online backup mode, backup mode will be terminated,
+        # rendering the backup useless.
+        #
+        # [1] https://www.postgresql.org/docs/9.1/server-shutdown.html
+        self.proc.send_signal(signal.SIGINT)
+        self.proc.wait()