Add plugins to make Nix more extensible.

shlevy · shlevy · commit c6597bd97db2 · 2018-02-08T11:26:18.000-05:00
See plugins.hh for the full details. Currently plugins to extend the
evaluator are supported.
diff --git a/.gitignore b/.gitignore
@@ -38,6 +38,7 @@ perl/Makefile.config
 /scripts/nix-copy-closure
 /scripts/nix-reduce-build
 /scripts/nix-http-export.cgi
+/scripts/nix-profile-daemon.sh
 
 # /src/libexpr/
 /src/libexpr/lexer-tab.cc
diff --git a/Makefile b/Makefile
@@ -24,7 +24,8 @@ makefiles = \
   misc/launchd/local.mk \
   misc/upstart/local.mk \
   doc/manual/local.mk \
-  tests/local.mk
+  tests/local.mk \
+  tests/plugins/local.mk
 
 GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include config.h
 
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
@@ -742,6 +742,19 @@ builtins.fetchurl {
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-plugin-files">
+    <term><literal>plugin-files</literal></term>
+    <listitem>
+      <para>
+        A list of plugin files to be loaded by Nix. This allows
+        extending nix in various ways; see the documentation in
+        <filename>plugins.hh</filename> for more information.
+      </para>
+    </listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </para>
diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml
@@ -389,6 +389,15 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     </para>
   </listitem>
 
+  <listitem>
+    <para>
+      Nix can now be extended with plugins. See the documentation in
+      <filename>plugins.hh</filename> for more details. Currently,
+      plugins to modify the Nix expression evaluator state are
+      supported.
+    </para>
+  </listitem>
+
 </itemizedlist>
 
 <para>Some features were removed:</para>
diff --git a/mk/libraries.mk b/mk/libraries.mk
@@ -45,6 +45,11 @@ endif
 # - $(1)_INSTALL_DIR: the directory where the library will be
 #   installed.  Defaults to $(libdir).
 #
+# - $(1)_EXCLUDE_FROM_LIBRARY_LIST: if defined, the library will not
+#   be automatically marked as a dependency of the top-level all
+#   target andwill not be listed in the make help output. This is
+#   useful for libraries built solely for testing, for example.
+#
 # - BUILD_SHARED_LIBS: if equal to ‘1’, a dynamic library will be
 #   built, otherwise a static library.
 define build-library
@@ -149,7 +154,9 @@ define build-library
   $(1)_DEPS := $$(foreach fn, $$($(1)_OBJS), $$(call filename-to-dep, $$(fn)))
   -include $$($(1)_DEPS)
 
+  ifndef $(1)_EXCLUDE_FROM_LIBRARY_LIST
   libs-list += $$($(1)_PATH)
+  endif
   clean-files += $$(_d)/*.a $$(_d)/*.$(SO_EXT) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
   dist-files += $$(_srcs)
 endef
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
@@ -17,6 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
+#include "plugins.hh"
 
 using namespace nix;
 using std::cin;
@@ -64,6 +65,8 @@ int main (int argc, char * * argv)
 
         settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work
 
+        initPlugins();
+
         auto store = openStore().cast<LocalStore>();
 
         /* It would be more appropriate to use $XDG_RUNTIME_DIR, since
diff --git a/src/buildenv/buildenv.cc b/src/buildenv/buildenv.cc
@@ -1,4 +1,5 @@
 #include "shared.hh"
+#include "plugins.hh"
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <fcntl.h>
@@ -135,6 +136,7 @@ int main(int argc, char ** argv)
 {
     return handleExceptions(argv[0], [&]() {
         initNix();
+        initPlugins();
         out = getEnv("out");
         if (mkdir(out.c_str(), 0755) == -1)
             throw SysError(format("creating %1%") % out);
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
@@ -323,6 +323,8 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     vEmptySet.type = tAttrs;
     vEmptySet.attrs = allocBindings(0);
 
+    /* This should always be last so plugins see a fully initialized
+       state, modulo builtins sorting. */
     createBaseEnv();
 }
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
@@ -210,13 +210,13 @@ private:
 
     void createBaseEnv();
 
+public:
+
     void addConstant(const string & name, Value & v);
 
     Value * addPrimOp(const string & name,
         unsigned int arity, PrimOpFun primOp);
 
-public:
-
     Value & getBuiltin(const string & name);
 
 private:
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
@@ -2228,6 +2228,9 @@ void EvalState::createBaseEnv()
         for (auto & primOp : *RegisterPrimOp::primOps)
             addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp));
 
+    for (const auto & plugin : settings.plugins.evalPlugins)
+        plugin(*this);
+
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
     baseEnv.values[0]->attrs->sort();
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
@@ -22,6 +22,7 @@ public:
 
 int handleExceptions(const string & programName, std::function<void()> fun);
 
+/* Don't forget to call initPlugins() after settings are initialized! */
 void initNix();
 
 void parseCmdLine(int argc, char * * argv,
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
@@ -3,6 +3,7 @@
 #include "types.hh"
 #include "config.hh"
 #include "util.hh"
+#include "plugins.hh"
 
 #include <map>
 #include <limits>
@@ -367,6 +368,11 @@ public:
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+
+    Setting<Paths> pluginFiles{this, {}, "plugin-files",
+        "Plugins to dynamically load at nix initialization time."};
+
+    Plugins plugins;
 };
 
 
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
@@ -9,6 +9,9 @@ libstore_SOURCES := $(wildcard $(d)/*.cc)
 libstore_LIBS = libutil libformat
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
+ifneq ($(OS), FreeBSD)
+ libstore_LDFLAGS += -ldl
+endif
 
 libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
 
diff --git a/src/libstore/plugins.cc b/src/libstore/plugins.cc
@@ -0,0 +1,51 @@
+#include "plugins.hh"
+#include "globals.hh"
+
+#include <dlfcn.h>
+
+namespace nix {
+
+static std::vector<Plugin> registeredPlugins;
+
+static void addPlugin(Plugin & p)
+{
+    switch (p.ty) {
+    case PluginType::EvalPlugin:
+        settings.plugins.evalPlugins.push_back(p.eval);
+        break;
+    }
+}
+
+void initPlugins()
+{
+    for (auto & plugin : registeredPlugins)
+        addPlugin(plugin);
+
+    for (const auto & pluginFile : settings.pluginFiles.get()) {
+        /* handle is purposefully leaked as there may be state in the
+           DSO needed by the action of the plugin. */
+        void *handle =
+            dlopen(pluginFile.c_str(), RTLD_LAZY | RTLD_LOCAL);
+        if (!handle)
+            throw Error(format("could not dynamically open plugin file '%1%': %2%") % pluginFile % dlerror());
+
+        dlerror();
+        auto plugins = (std::vector<Plugin> *) dlsym(handle, "plugins");
+        if (!plugins) {
+            auto err = dlerror();
+            if (err)
+                throw Error(format("could not load plugins from plugin file '%1%': %2%") % pluginFile % err);
+            else
+                throw Error(format("plugins symbol from plugin '%1%' unexpectedly resolved to NULL") % pluginFile);
+        }
+        for (auto & plugin : *plugins)
+            addPlugin(plugin);
+    }
+}
+
+RegisterPlugin::RegisterPlugin(Plugin p)
+{
+    registeredPlugins.emplace_back(std::move(p));
+}
+
+}
diff --git a/src/libstore/plugins.hh b/src/libstore/plugins.hh
@@ -0,0 +1,75 @@
+#pragma once
+
+#include "types.hh"
+#include <vector>
+
+/* Plugin support for nix
+ *
+ * This file defines interfaces for extending nix with plugins.
+ * Plugins can be registered in two ways:
+ *
+ *   1. Construct a nix::RegisterPlugin before the call to
+ *      initPlugins(), usually as a static initialization. This can be
+ *      used by nix itself, external programs using libnixmain
+ *      directly, or LD_PRELOADed libraries.
+ *   2. Add a file to the 'plugin-files' nix configuration setting. It
+ *      is expected that that file can be safely dlopened by nix (i.e.
+ *      it is a DSO compiled against the same headers as the running
+ *      nix instance that is not linked to any shared libraries
+ *      incompatible with the running nix instance) and contains a
+ *      symbol named "plugins" referencing a std::vector<Plugin>, each
+ *      of which is registered. Note that to better ensure
+ *      compatibility the DSO should *not* be linked to any of nix's
+ *      libraries. See tests/plugins/plugintest.cc for a dumb example.
+ *
+ * There is no ABI compatibility guarantee for plugins across
+ * different versions or even configurations of nix. The source API
+ * will be kept compatible in accordance with semver.
+ */
+
+namespace nix {
+
+struct EvalState;
+
+/* The types of plugins nix supports */
+enum class PluginType {
+    EvalPlugin = 0
+};
+
+/* An EvalPlugin is called after the evaluation state is initialized
+ *except* that the builtins attribute set is not yet sorted. Thus,
+ EvalPlugins can define new primops without re-sorting builtins, but
+ they must not assume the expected attribute set invariants hold for
+ builtins during the call of the EvalPlugin itself! */
+typedef void (*EvalPlugin)(EvalState & state);
+
+template <PluginType t> struct PluginTag {};
+
+struct Plugin {
+    PluginType ty;
+    union {
+        EvalPlugin eval;
+    };
+    Plugin(PluginTag<PluginType::EvalPlugin>, EvalPlugin eval) :
+        ty{PluginType::EvalPlugin}, eval{eval} {}
+};
+
+struct Plugins {
+    std::vector<EvalPlugin> evalPlugins;
+};
+
+/* Initialize all registered plugins. This should be called *after*
+   all nix config options are initialized (e.g. after initNix() and
+   parseCmdLine() for a normal nix command line tool) but before any
+   other initialization. */
+void initPlugins();
+
+struct RegisterPlugin
+{
+    /* Register a plugin. This must be called *before* initPlugins(),
+       usually in a static initializer. Plugins in the 'plugin-files'
+       list should *not* use this interface! */
+    RegisterPlugin(Plugin p);
+};
+
+}
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
@@ -232,6 +232,8 @@ void mainWrapped(int argc, char * * argv)
 
     myArgs.parseCmdline(args);
 
+    initPlugins();
+
     if (packages && fromArgs)
         throw UsageError("'-p' and '-E' are mutually exclusive");
 
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
@@ -213,6 +213,9 @@ int main(int argc, char ** argv)
             }
             return true;
         });
+
+        initPlugins();
+
         switch (cmd) {
             case cNone:
                 throw UsageError("no command specified");
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -77,6 +77,8 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        initPlugins();
+
         auto profilesDir = settings.nixStateDir + "/profiles";
         if (removeOld) removeOldGenerations(profilesDir);
 
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
@@ -44,6 +44,8 @@ int main(int argc, char ** argv)
             return true;
         });
 
+        initPlugins();
+
         if (sshHost.empty())
             throw UsageError("no host name specified");
 
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
@@ -1060,6 +1060,8 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        initPlugins();
+
         if (stdio) {
             if (getStoreType() == tDaemon) {
                 /* Forward on this connection to the real daemon */
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
@@ -1393,6 +1393,8 @@ int main(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
+        initPlugins();
+
         if (!op) throw UsageError("no operation specified");
 
         auto store = openStore();
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
@@ -151,6 +151,8 @@ int main(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
+        initPlugins();
+
         if (evalOnly && !wantsReadWrite)
             settings.readOnlyMode = true;
 
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -89,6 +89,8 @@ int main(int argc, char * * argv)
 
         myArgs.parseCmdline(argvToStrings(argc, argv));
 
+        initPlugins();
+
         if (args.size() > 2)
             throw UsageError("too many arguments");
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
@@ -1052,6 +1052,8 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        initPlugins();
+
         if (!op) throw UsageError("no operation specified");
 
         if (op != opDump && op != opRestore) /* !!! hack */
diff --git a/src/nix/main.cc b/src/nix/main.cc
@@ -92,6 +92,8 @@ void mainWrapped(int argc, char * * argv)
 
     args.parseCmdline(argvToStrings(argc, argv));
 
+    initPlugins();
+
     if (!args.command) args.showHelpAndExit();
 
     Finally f([]() { stopProgressBar(); });
diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
diff --git a/tests/local.mk b/tests/local.mk
diff --git a/tests/plugins.sh b/tests/plugins.sh
diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk
diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc

Original file line number	Diff line number	Diff line change
`@@ -323,6 +323,8 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)`
`323`	`323`	`vEmptySet.type = tAttrs;`
`324`	`324`	`vEmptySet.attrs = allocBindings(0);`
`325`	`325`
	`326`	`+ /* This should always be last so plugins see a fully initialized`
	`327`	`+ state, modulo builtins sorting. */`
`326`	`328`	`createBaseEnv();`
`327`	`329`	`}`
`328`	`330`