first version of lib-autopilot.py

contrib/autopilot/bech32.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2017 Pieter Wuille
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+"""subset of the reference implementation for Bech32 addresses."""
+
+CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"
+
+def bech32_polymod(values):
+    """Internal function that computes the Bech32 checksum."""
+    generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3]
+    chk = 1
+    for value in values:
+        top = chk >> 25
+        chk = (chk & 0x1ffffff) << 5 ^ value
+        for i in range(5):
+            chk ^= generator[i] if ((top >> i) & 1) else 0
+    return chk
+
+
+def bech32_hrp_expand(hrp):
+    """Expand the HRP into values for checksum computation."""
+    return [ord(x) >> 5 for x in hrp] + [0] + [ord(x) & 31 for x in hrp]
+
+
+def bech32_verify_checksum(hrp, data):
+    """Verify a checksum given HRP and converted data characters."""
+    return bech32_polymod(bech32_hrp_expand(hrp) + data) == 1
+
+
+def bech32_decode(bech):
+    """Validate a Bech32 string, and determine HRP and data."""
+    if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or
+            (bech.lower() != bech and bech.upper() != bech)):
+        return (None, None)
+    bech = bech.lower()
+    pos = bech.rfind('1')
+    if pos < 1 or pos + 7 > len(bech) or len(bech) > 90:
+        return (None, None)
+    if not all(x in CHARSET for x in bech[pos+1:]):
+        return (None, None)
+    hrp = bech[:pos]
+    data = [CHARSET.find(x) for x in bech[pos+1:]]
+    if not bech32_verify_checksum(hrp, data):
+        return (None, None)
+    return (hrp, data[:-6])
+
+
+def convertbits(data, frombits, tobits, pad=True):
+    """General power-of-2 base conversion."""
+    acc = 0
+    bits = 0
+    ret = []
+    maxv = (1 << tobits) - 1
+    max_acc = (1 << (frombits + tobits - 1)) - 1
+    for value in data:
+        if value < 0 or (value >> frombits):
+            return None
+        acc = ((acc << frombits) | value) & max_acc
+        bits += frombits
+        while bits >= tobits:
+            bits -= tobits
+            ret.append((acc >> bits) & maxv)
+    if pad:
+        if bits:
+            ret.append((acc << (tobits - bits)) & maxv)
+    elif bits >= frombits or ((acc << (tobits - bits)) & maxv):
+        return None
+    return ret

contrib/autopilot/c-lightning-autopilot.py

@@ -0,0 +1,269 @@
+'''
+Created on 04.09.2018
+
+@author: rpickhardt
+
+This software is a command line tool and c-lightning wrapper for lib_autopilot
+
+You need to have a c-lightning node running in order to utilize this program.
+Also you need lib_autopilot. You can run
+
+python3 c-lightning-autopilot --help
+
+in order to get all the command line options
+
+usage: c-lightning-autopilot.py [-h] [-b BALANCE] [-c CHANNELS]
+                                [-r PATH_TO_RPC_INTERFACE]
+                                [-s {diverse,merge}] [-p PERCENTILE_CUTOFF]
+                                [-d] [-i INPUT]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -b BALANCE, --balance BALANCE
+                        use specified number of satoshis to open all channels
+  -c CHANNELS, --channels CHANNELS
+                        opens specified amount of channels
+  -r PATH_TO_RPC_INTERFACE, --path_to_rpc_interface PATH_TO_RPC_INTERFACE
+                        specifies the path to the rpc_interface
+  -s {diverse,merge}, --strategy {diverse,merge}
+                        defines the strategy
+  -p PERCENTILE_CUTOFF, --percentile_cutoff PERCENTILE_CUTOFF
+                        only uses the top percentile of each probability
+                        distribution
+  -d, --dont_store      don't store the network on the hard drive
+  -i INPUT, --input INPUT
+                        points to a pickle file
+
+a good example call of the program could look like that: 
+
+python3 c-lightning-autopilot.py -s diverse -c 30 -b 10000000 
+
+This call would use up to 10'000'000 satoshi to create 30 channels which are
+generated by using the diverse strategy to mix the 4 heuristics. 
+
+Currently the software will not check, if sufficient funds are available
+or if a channel already exists.
+'''
+
+from os.path import expanduser
+import argparse
+import logging
+import math
+import pickle
+import sys
+
+from lightning import LightningRpc
+import dns.resolver
+
+from bech32 import bech32_decode, CHARSET, convertbits
+from lib_autopilot import Autopilot
+from lib_autopilot import Strategy
+import networkx as nx
+
+
+class CLightning_autopilot(Autopilot):
+
+    def __init__(self, path, input=None, dont_store=None):
+        self.__add_clogger()
+
+        self.__rpc_interface = LightningRpc(path)
+        self.__clogger.info("connection to RPC interface successful")
+
+        G = None
+        if input:
+            try:
+                self.__clogger.info(
+                    "Try to load graph from file system at:" + input)
+                with open(input, "rb") as infile:
+                    G = pickle.load(infile)
+                    self.__clogger.info(
+                        "Successfully restored the lightning network graph from data/networkx_graph")
+            except FileNotFoundError:
+                self.__clogger.info(
+                    "input file not found. Load the graph from the peers of the lightning network")
+                G = self.__download_graph()
+        else:
+            self.__clogger.info(
+                    "no input specified download graph from peers")
+            G = self.__download_graph()
+
+        if dont_store is None:          
+            with open("lightning_networkx_graph.pickle", "wb") as outfile:
+                    pickle.dump(G, outfile, pickle.HIGHEST_PROTOCOL)
+
+        Autopilot.__init__(self,G)
+
+
+
+    def __add_clogger(self):
+        """ initiates the logging service for this class """
+        # FIXME: adapt to the settings that are proper for you
+        self.__clogger = logging.getLogger('clightning-autopilot')
+        self.__clogger.setLevel(logging.INFO)
+        ch = logging.StreamHandler()
+        ch.setLevel(logging.INFO)
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        ch.setFormatter(formatter)
+        self.__clogger.addHandler(ch)
+        self.__clogger.info("set up logging infrastructure")
+
+    def __get_seed_keys(self):
+        """
+        retrieve the nodeids of the ln seed nodes from lseed.bitcoinstats.com
+        """
+        domain = "lseed.bitcoinstats.com"
+        srv_records = dns.resolver.query(domain,"SRV")
+        res = []
+        for srv in srv_records:
+            bech32 = str(srv.target).rstrip(".").split(".")[0]
+            data = bech32_decode(bech32)[1]
+            decoded = convertbits(data, 5, 4)
+            res.append("".join(
+                ['{:1x}'.format(integer) for integer in decoded])[:-1])
+        return res
+
+
+    def __connect_to_seeds(self):
+        """
+        sets up peering connection to seed nodes of the lightning network
+
+        This is necessary in case the node operating the autopilot has never
+        been connected to the lightning network.
+        """
+        try:
+            for nodeid in random.shuffle(self.__get_seed_keys()):
+                self.__clogger.info("peering with node: " + nodeid)
+                self.__rpc_interface.connect(nodeid)
+                # FIXME: better strategy than sleep(2) for building up
+                time.sleep(2)
+        except:
+            pass
+
+    def __download_graph(self):
+        """
+        Downloads a local copy of the nodes view of the lightning network
+
+        This copy is retrieved by listnodes and listedges RPC calls and will
+        thus be incomplete as peering might not be ready yet.
+        """
+
+        # FIXME: it is a real problem that we don't know how many nodes there
+        # could be. In particular billion nodes networks will outgrow memory
+        G = nx.Graph()
+        self.__clogger.info("Instantiated networkx graph to store the lightning network")
+
+        nodes = []
+        try:
+            self.__clogger.info(
+                "Attempt RPC-call to download nodes from the lightning network")
+            while len(nodes) == 0:
+                peers = self.__rpc_interface.listpeers()["peers"]
+                if len(peers) < 1:
+                    self.__connect_to_seeds()
+                nodes = self.__rpc_interface.listnodes()["nodes"]
+        except ValueError as e:
+            self.__clogger.info(
+                "Node list could not be retrieved from the peers of the lightning network")
+            self.__clogger.debug("RPC error: " + str(e))
+            raise e 
+
+        for node in nodes:
+            G.add_node(node["nodeid"], **node)
+
+        self.__clogger.info(
+            "Number of nodes found and added to the local networkx graph: {}".format(len(nodes)))
+
+
+        channels = {}
+        try:
+            self.__clogger.info(
+                "Attempt RPC-call to download channels from the lightning network")
+            channels = self.__rpc_interface.listchannels()["channels"]
+            self.__clogger.info(
+                "Number of retrieved channels: {}".format(
+                    len(channels)))
+        except ValueError as e:
+            self.__clogger.info(
+                "Channel list could not be retrieved from the peers of the lightning network")
+            self.__clogger.debug("RPC error: " + str(e))
+            return False
+
+        for channel in channels:
+            G.add_edge(
+                channel["source"],
+                channel["destination"],
+                **channel)
+
+        return G
+
+    def connect(self, candidates, balance=1000000):
+        pdf = self.calculate_statistics(candidates)
+        connection_dict = self.calculate_proposed_channel_capacities(
+            pdf, balance)
+        for nodeid, fraction in connection_dict.items():
+            try:
+                satoshis = math.ceil(balance * fraction)
+                self.__clogger.info(
+                    "Try to open channel with a capacity of {} to node {}".format(
+                        satoshis, nodeid))
+                self.__rpc_interface.fundchannel(nodeid, satoshis)
+            except ValueError as e:
+                self.__clogger.info(
+                    "Could not open a channel to {} with capacity of {}. Error: {}".format(
+                        nodeid, satoshis, str(e)))        
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-b", "--balance", 
+                        help="use specified number of satoshis to open all channels")
+    parser.add_argument("-c", "--channels", 
+                        help="opens specified amount of channels")
+    # FIXME: add the following command line option
+    # parser.add_argument("-m", "--maxchannels", 
+    #                help="opens channels as long as maxchannels is not reached")
+    parser.add_argument("-r", "--path_to_rpc_interface", 
+                    help="specifies the path to the rpc_interface")
+    parser.add_argument("-s", "--strategy",choices=[Strategy.DIVERSE,Strategy.MERGE],
+                        help = "defines the strategy ")
+    parser.add_argument("-p", "--percentile_cutoff",
+                        help = "only uses the top percentile of each probability distribution")
+    parser.add_argument("-d", "--dont_store", action='store_true',
+                        help = "don't store the network on the hard drive")
+    parser.add_argument("-i", "--input",
+                        help = "points to a pickle file")
+
+
+
+    args = parser.parse_args()
+
+    # FIXME: find ln-dir from lightningd.
+    path = path = expanduser("~/.lightning/lightning-rpc")
+    if args.path_to_rpc_interface is not None:
+        path=expanduser(parser.path-to-rpc-interface)
+
+    balance = 1000000
+    if args.balance is not None:
+        # FIXME: parser.argument does not accept type = int
+        balance = int(args.balance)
+
+    num_channels = 21
+    if args.channels is not None:
+        # FIXME: parser.argument does not accept type = int
+        num_channels = int(args.channels)
+
+    percentile = None
+    if args.percentile_cutoff is not None:
+        # FIXME: parser.argument does not accept type = float
+        percentile = float(args.percentile_cutoff)
+
+    autopilot = CLightning_autopilot(path, input = args.input,
+                                     dont_store = args.dont_store)
+
+    candidates = autopilot.find_candidates(num_channels,
+                                           strategy = args.strategy, 
+                                           percentile = percentile)
+
+    autopilot.connect(candidates, balance)
+    print("Autopilot finished. We hope it did a good job for you (and the lightning network). Thanks for using it.")