clean up linkage of libsecp256k1 into libwallycore and Python extension

CHANGES.md

@@ -1,5 +1,16 @@
 # Changes
 
+## Version 1.0.0
+- The library now follows semantic versioning as per https://semver.org/.
+- Elements support is now enabled by default, reflecting the common library
+  usage. Please see the `configure --help` entries for `--disable-elements`
+  and `--disable-elements-abi` for details.
+- The ABI of the library is now consistent by default regardless of whether
+  it is built with or without Elements support.
+- Some functions in the c++ header wally.hpp have changed interface slightly.
+  Note that this header is deprecated and will be replaced in an upcoming
+  release with higher level wrappers in the same manner as Python and JS.
+
 ## Version 0.9.1
 - PSET: When adding an Elements transaction output to a PSET, the nonce
   commitment was incorrectly mapped to the PSET output blinding key field.

README.md

@@ -5,7 +5,7 @@ for cryptocurrency wallets.
 
 Read the API documentation at https://wally.readthedocs.io.
 
-Note that library interfaces may change slightly while the library design matures. Please see the [CHANGES](./CHANGES.md) file to determine if the API has changed when upgrading.
+Please see the [CHANGES](./CHANGES.md) for details of ABI changes when upgrading.
 
 Please report bugs and submit patches to [Our github repository](https://github.com/ElementsProject/libwally-core). If you wish to report a security issue, please read [Our security reporting guidelines](./SECURITY.md).
 
@@ -117,7 +117,7 @@ installed.
 For non-development use, you can install wally with `pip` as follows:
 
 ```
-pip install wallycore==0.9.2
+pip install wallycore==1.0.0
 ```
 
 For python development, you can build and install wally using:

_CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.20)
 
 project(
   libwallycore
-  VERSION 0.9.2
+  VERSION 1.0.0
   DESCRIPTION "A collection of useful primitives for cryptocurrency wallets"
   LANGUAGES C
 )

configure.ac

@@ -1,5 +1,5 @@
 AC_PREREQ([2.60])
-AC_INIT([libwallycore],[0.9.2])
+AC_INIT([libwallycore],[1.0.0])
 AC_CONFIG_AUX_DIR([tools/build-aux])
 AC_CONFIG_MACRO_DIR([tools/build-aux/m4])
 AC_CONFIG_SRCDIR([src/mnemonic.h])
@@ -259,26 +259,10 @@ fi
 #
 # libsecp256k1
 #
-# FIXME: This is needed to force libtool to use all object files from secp.
-#        We can only build secp properly by recursively invoking
-#        configure/make, and can't include it as a noinst_ library. Libtool
-#        assumes that such libraries will be installed along with our library
-#        target and so won't force all object files in the library to be
-#        included in ours - despite the fact that we are making a shared
-#        library and linking to a static one. This is broken and we work
-#        around it by hacking the secp objects directly into the library
-#        via the _LDADD variable for wallycore.
-#        We previously achieved this by adding the libsecp256k1.a archive,
-#        but changes to libtool and apples linkers mean that
-#        archives-within-archives no longer work.
-#        Because automake tries to police its users very strictly and fails
-#        hard when flags are passed in this way, we have to substitute the
-#        flags here.
-#        Because libtool both intercepts -Wl and arbitrarily re-orders its
-#        command line inputs, we have to concoct a single expression to
-#        enforce linking that cannot be split, hence the below expression.
-LIBADD_SECP256K1="-Wl,secp256k1/src/libsecp256k1_la-secp256k1.${OBJEXT},secp256k1/src/libsecp256k1_precomputed_la-precomputed_ecmult_gen.${OBJEXT},secp256k1/src/libsecp256k1_precomputed_la-precomputed_ecmult.${OBJEXT}"
-AC_SUBST([LIBADD_SECP256K1])
+libsecp256k1_CFLAGS='-I$(top_srcdir)/src/secp256k1/include'
+libsecp256k1_LIBS='$(top_srcdir)/src/secp256k1/libsecp256k1.la'
+AC_SUBST([libsecp256k1_CFLAGS])
+AC_SUBST([libsecp256k1_LIBS])
 
 #
 # Python facilities
@@ -404,8 +388,7 @@ export ARFLAGS
 export AR_FLAGS
 export LD
 export LDFLAGS
-ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-experimental --enable-module-ecdh --enable-module-recovery --enable-module-ecdsa-s2c --enable-module-rangeproof --enable-module-surjectionproof --enable-module-whitelist --enable-module-generator --enable-module-extrakeys --enable-module-schnorrsig ${secp256k1_test_opt} --enable-exhaustive-tests=no --enable-benchmark=no --disable-dependency-tracking ${secp_asm}"
-AC_CONFIG_SUBDIRS([src/secp256k1])
 
+AX_SUBDIRS_CONFIGURE([src/secp256k1], [[--disable-shared], [--enable-static], [--with-pic], [--enable-experimental], [--enable-module-ecdh], [--enable-module-recovery], [--enable-module-ecdsa-s2c], [--enable-module-rangeproof], [--enable-module-surjectionproof], [--enable-module-whitelist], [--enable-module-generator], [--enable-module-extrakeys], [--enable-module-schnorrsig], [$secp256k1_test_opt], [--enable-exhaustive-tests=no], [--enable-benchmark=no], [--disable-dependency-tracking], [$secp_asm]])
 
 AC_OUTPUT

docs/source/conf.py

@@ -167,7 +167,7 @@ def extract_docs(infile, outfile):
 # built documents.
 #
 # The short X.Y version.
-version = u'0.9.2'
+version = u'1.0.0'
 # The full version, including alpha/beta/rc tags.
 release = version
 

include/wally_core.h

@@ -29,10 +29,10 @@ extern "C" {
 #define WALLY_ENOMEM -3 /** malloc() failed */
 
 /** Library version */
-#define WALLY_MAJOR_VER 0
-#define WALLY_MINOR_VER 9
-#define WALLY_PATCH_VER 2
-#define WALLY_BUILD_VER 0x902
+#define WALLY_MAJOR_VER 1
+#define WALLY_MINOR_VER 0
+#define WALLY_PATCH_VER 0
+#define WALLY_BUILD_VER 0x10000
 
 /**
  * Initialize wally.

setup.py

@@ -3,8 +3,15 @@
 import copy, os, platform, shutil
 import distutils.sysconfig
 
-CONFIGURE_ARGS = '--enable-swig-python --enable-python-manylinux'
-CONFIGURE_ARGS += ' --disable-swig-java --disable-tests --disable-dependency-tracking'
+build_shared = os.environ.get('WALLY_ABI_PY_WHEEL_USE_DSO', '').lower() not in ['', '0', 'false', 'no', 'n', 'off']
+if build_shared:
+    CONFIGURE_ARGS = ['--enable-shared', '--disable-static']
+else:
+    CONFIGURE_ARGS = ['--disable-shared', '--enable-static', '--with-pic']
+
+CONFIGURE_ARGS += [
+    '--enable-swig-python', '--enable-python-manylinux',
+    '--disable-swig-java', '--disable-tests', '--disable-dependency-tracking']
 
 distutils_env = distutils.sysconfig.get_config_vars()
 configure_env = copy.deepcopy(os.environ)
@@ -26,11 +33,11 @@
     if is_x86 and not is_native:
         # We are cross-compiling or compiling a univeral2 binary.
         # Configure our source code as a cross compile to make the build work
-        CONFIGURE_ARGS += ' --host x86_64-apple-darwin'
+        CONFIGURE_ARGS += ['--host', 'x86_64-apple-darwin']
         arch = 'universal2' if len(archs) > 1 else archs[0]
-        CONFIGURE_ARGS += ' --target {}-apple-macos'.format(arch)
+        CONFIGURE_ARGS += ['--target', '{}-apple-macos'.format(arch)]
         if len(archs) > 1:
-            CONFIGURE_ARGS += ' --with-asm=no'
+            CONFIGURE_ARGS += ['--with-asm=no']
         if 'PY_CFLAGS' in distutils_env:
             configure_env['CFLAGS'] = distutils_env['PY_CFLAGS']
             configure_env['LDFLAGS'] = distutils_env['PY_LDFLAGS']
@@ -40,18 +47,17 @@
     # then build using the standard Python ext module machinery.
     # (Windows requires source generation to be done separately).
     import multiprocessing
-    import os
     import subprocess
 
     abs_path = os.path.dirname(os.path.abspath(__file__)) + '/'
 
-    def call(cmd):
-        subprocess.check_call(cmd.split(' '), cwd=abs_path, env=configure_env)
+    def call(args):
+        subprocess.check_call(args, cwd=abs_path, env=configure_env)
 
-    call('./tools/cleanup.sh')
-    call('./tools/autogen.sh')
-    call('./configure {}'.format(CONFIGURE_ARGS))
-    call('make -j{}'.format(multiprocessing.cpu_count()))
+    call(['./tools/cleanup.sh'])
+    call(['./tools/autogen.sh'])
+    call(['./configure'] + CONFIGURE_ARGS)
+    call(['make', '-j{}'.format(multiprocessing.cpu_count())])
 
 define_macros=[
     ('SWIG_PYTHON_BUILD', None),
@@ -67,33 +73,34 @@ def call(cmd):
 include_dirs=[
     './',
     './src',
-    './include',
-    './src/ccan',
-    './src/secp256k1',
-    './src/secp256k1/src/'
+    './src/secp256k1/include',
     ]
 if is_windows:
     shutil.copyfile('./src/amalgamation/windows_config/libsecp256k1-config.h', 'src/secp256k1/src/libsecp256k1-config.h')
-    include_dirs = ['./src/amalgamation/windows_config'] + include_dirs
+    include_dirs = ['./src/amalgamation/windows_config'] + include_dirs + ['./src/ccan']
 
 extra_compile_args = ['-flax-vector-conversions']
 
 wally_ext = Extension(
     '_wallycore',
     define_macros=define_macros,
     include_dirs=include_dirs,
+    library_dirs=['src/.libs'] + ([] if build_shared else ['src/secp256k1/.libs']),
+    libraries=['wallycore'] + ([] if build_shared else ['secp256k1']),
     extra_compile_args=extra_compile_args,
     sources=[
-        'src/swig_python/swig_wrap.c' if is_windows else 'src/swig_python/swig_python_wrap.c',
+        'src/swig_python/swig_wrap.c',
         'src/amalgamation/combined.c',
         'src/amalgamation/combined_ccan.c',
         'src/amalgamation/combined_ccan2.c',
+        ] if is_windows else [
+        'src/swig_python/swig_python_wrap.c',
         ],
     )
 
 kwargs = {
     'name': 'wallycore',
-    'version': '0.9.2',
+    'version': '1.0.0',
     'description': 'libwally Bitcoin library',
     'long_description': 'Python bindings for the libwally Bitcoin library',
     'url': 'https://github.com/ElementsProject/libwally-core',

src/Makefile.am

@@ -40,7 +40,7 @@ if USE_SWIG_PYTHON
 noinst_LTLIBRARIES += libswig_python.la
 libswig_python_la_SOURCES = swig_python/swig_python_wrap.c
 
-libswig_python_la_CFLAGS = -I$(top_srcdir) $(AM_CFLAGS) $(SWIG_PYTHON_CPPFLAGS) $(SWIG_WARN_CFLAGS) $(NOALIAS_CFLAGS)
+libswig_python_la_CFLAGS = -I$(top_srcdir) $(libsecp256k1_CFLAGS) $(AM_CFLAGS) $(SWIG_PYTHON_CPPFLAGS) $(SWIG_WARN_CFLAGS) $(NOALIAS_CFLAGS)
 if PYTHON_MANYLINUX
 else
 libswig_python_la_LIBADD = $(PYTHON_LIBS)
@@ -83,7 +83,7 @@ noinst_LTLIBRARIES += libswig_java.la
 libswig_java_la_SOURCES = \
     swig_java/swig_java_wrap.c
 
-libswig_java_la_CFLAGS = -I$(top_srcdir) $(AM_CFLAGS) $(SWIG_JAVA_CPPFLAGS) $(SWIG_WARN_CFLAGS)
+libswig_java_la_CFLAGS = -I$(top_srcdir) $(libsecp256k1_CFLAGS) $(AM_CFLAGS) $(SWIG_JAVA_CPPFLAGS) $(SWIG_WARN_CFLAGS)
 
 SWIG_JOPT = $(SWIG_JAVA_OPT) -outdir swig_java -noproxy -package com.blockstream.libwally
 
@@ -207,8 +207,8 @@ libwallycore_la_LDFLAGS += -no-undefined
 endif
 endif # SHARED_BUILD_ENABLED
 
-libwallycore_la_CFLAGS = -I$(top_srcdir) -I$(srcdir)/ccan -DWALLY_CORE_BUILD=1 $(AM_CFLAGS)
-libwallycore_la_LIBADD = $(LIBADD_SECP256K1) $(noinst_LTLIBRARIES)
+libwallycore_la_CFLAGS = -I$(top_srcdir) -I$(srcdir)/ccan $(libsecp256k1_CFLAGS) -DWALLY_CORE_BUILD=1 $(AM_CFLAGS)
+libwallycore_la_LIBADD = $(libsecp256k1_LIBS) $(noinst_LTLIBRARIES)
 
 SUBDIRS = secp256k1
 

src/ecdh.c

@@ -1,7 +1,7 @@
 #include "internal.h"
 #include <include/wally_crypto.h>
-#include "secp256k1/include/secp256k1.h"
-#include "secp256k1/include/secp256k1_ecdh.h"
+#include <secp256k1.h>
+#include <secp256k1_ecdh.h>
 
 int wally_ecdh(const unsigned char *pub_key, size_t pub_key_len,
                const unsigned char *priv_key, size_t priv_key_len,

src/elements.c

@@ -7,10 +7,10 @@
 #include <include/wally_crypto.h>
 #include <include/wally_symmetric.h>
 #include <include/wally_transaction.h>
-#include "secp256k1/include/secp256k1_generator.h"
-#include "secp256k1/include/secp256k1_rangeproof.h"
-#include "src/secp256k1/include/secp256k1_surjectionproof.h"
-#include "src/secp256k1/include/secp256k1_whitelist.h"
+#include <secp256k1_generator.h>
+#include <secp256k1_rangeproof.h>
+#include <secp256k1_surjectionproof.h>
+#include <secp256k1_whitelist.h>
 
 
 static const unsigned char LABEL_STR[] = {

src/internal.h

@@ -13,11 +13,11 @@
 #pragma GCC diagnostic ignored "-Wunused-parameter"
 #endif /* BUILD_ELEMENTS */
 #endif /* WALLY_ABI_NO_ELEMENTS */
-#include "secp256k1/include/secp256k1.h"
-#include "secp256k1/include/secp256k1_recovery.h"
-#include "secp256k1/include/secp256k1_extrakeys.h"
+#include <secp256k1.h>
+#include <secp256k1_recovery.h>
+#include <secp256k1_extrakeys.h>
 #ifndef BUILD_STANDARD_SECP
-#include "secp256k1/include/secp256k1_ecdsa_s2c.h"
+#include <secp256k1_ecdsa_s2c.h>
 #endif
 #include <config.h>
 #if defined(HAVE_MEMSET_S)

src/sign.c

@@ -1,7 +1,7 @@
 #include "internal.h"
 #include <include/wally_crypto.h>
 #include "script_int.h"
-#include "secp256k1/include/secp256k1_schnorrsig.h"
+#include <secp256k1_schnorrsig.h>
 #include "ccan/ccan/build_assert/build_assert.h"
 
 #define EC_FLAGS_TYPES (EC_FLAG_ECDSA | EC_FLAG_SCHNORR)

src/wasm_package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wallycore",
-  "version": "0.9.2",
+  "version": "1.0.0",
   "description": "JavaScript bindings for libwally",
   "main": "src/index.js",
   "type": "module",

src/wasm_package/src/const.js

@@ -109,7 +109,7 @@ export const WALLY_ADDRESS_VERSION_WIF_TESTNET = 0xEF; /** Wallet Import Format
 export const WALLY_BIP32_CHAIN_CODE_LEN = 32;
 export const WALLY_BIP32_TWEAK_SUM_LEN = 32;
 export const WALLY_BTC_MAX = 21000000;
-export const WALLY_BUILD_VER = 0x902;
+export const WALLY_BUILD_VER = 0x10000;
 export const WALLY_CA_PREFIX_LIQUID = 0x0c; /** Liquid v1 confidential address prefix */
 export const WALLY_CA_PREFIX_LIQUID_REGTEST = 0x04; /** Liquid v1 confidential address prefix for regtest */
 export const WALLY_CA_PREFIX_LIQUID_TESTNET = 0x17; /** Liquid v1 confidential address prefix for testnet */
@@ -118,14 +118,14 @@ export const WALLY_EINVAL = -2; /** Invalid argument */
 export const WALLY_ENOMEM = -3; /** malloc() failed */
 export const WALLY_ERROR = -1; /** General error */
 export const WALLY_HOST_COMMITMENT_LEN = 32;
-export const WALLY_MAJOR_VER = 0;
+export const WALLY_MAJOR_VER = 1;
 export const WALLY_MAX_OP_RETURN_LEN = 80; /* Maximum length of OP_RETURN data push */
 export const WALLY_MINISCRIPT_DEPTH_MASK = 0xffff0000; /** Mask for limiting maximum depth */
 export const WALLY_MINISCRIPT_DEPTH_SHIFT = 16; /** Shift to convert maximum depth to flags */
 export const WALLY_MINISCRIPT_ONLY = 0x02; /** Only allow miniscript (not descriptor) expressions */
 export const WALLY_MINISCRIPT_REQUIRE_CHECKSUM = 0x04; /** Require a checksum to be present */
 export const WALLY_MINISCRIPT_TAPSCRIPT = 0x01; /** Tapscript, use x-only pubkeys */
-export const WALLY_MINOR_VER = 9;
+export const WALLY_MINOR_VER = 0;
 export const WALLY_MS_CANONICAL_NO_CHECKSUM = 0x01; /** Do not include a checksum */
 export const WALLY_MS_IS_DESCRIPTOR = 0x20; /** Contains only descriptor expressions (no miniscript) */
 export const WALLY_MS_IS_MULTIPATH = 0x02; /** Allows multiple paths via ``<a;b;c>`` */
@@ -142,7 +142,7 @@ export const WALLY_NETWORK_LIQUID_TESTNET = 0x05; /** Liquid v1 testnet */
 export const WALLY_NETWORK_NONE = 0x00; /** Used for miniscript parsing only */
 export const WALLY_NO_CODESEPARATOR = 0xffffffff; /* No BIP342 code separator position */
 export const WALLY_OK = 0; /** Success */
-export const WALLY_PATCH_VER = 2;
+export const WALLY_PATCH_VER = 0;
 export const WALLY_PSBT_EXTRACT_NON_FINAL = 0x1; /* Extract without final scriptsig and witness */
 export const WALLY_PSBT_FINALIZE_NO_CLEAR = 0x1; /* Finalize without clearing redeem/witness scripts etc */
 export const WALLY_PSBT_FLAG_NON_FINAL = 0x1;

tools/autogen.sh

@@ -8,3 +8,7 @@ if uname | grep "Darwin" >/dev/null 2>&1; then
         done
     done
 fi
+
+if [ -x src/secp256k1/autogen.sh ] ; then
+    cd src/secp256k1 && ./autogen.sh
+fi