diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e2f5dd2e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +result \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..19735b27 --- /dev/null +++ b/README.md @@ -0,0 +1,112 @@ +# Repl.it Nix Modules + +This repository holds Repl.it's Nix modules. + +* Each module is located as a folder under `pkgs/modules` +* `pkgs/modules/default.nix` specifies a list of the active modules, some of which are parameterized with the version of the runtime or compiler + +To list all active modules, you can do: + +``` +nix eval .#modules --json | jq +``` +Output might look like: +```json +{ + "bun-0.5-m1.0": "/nix/store/nqwdhvs2n6fv82nkn77rg5wb3g1giwjs-replit-module-bun-0.5-m1.0", + "c-14.0-m1.0": "/nix/store/c8v74sbwivlj0mridwsdng883frwccy5-replit-module-c-14.0-m1.0", + "clojure-1.11-m1.0": "/nix/store/dsx4w6inr69f6c799qija5nf08q1ds39-replit-module-clojure-1.11-m1.0", + "cpp-14.0-m1.0": "/nix/store/065sbbghckzyirbxq239s4y3bcsr2wq2-replit-module-cpp-14.0-m1.0", + "dart-2.18-m1.0": "/nix/store/ampynbffmbjckc4xqzndlhbxncdljqfv-replit-module-dart-2.18-m1.0", + "dotnet-7.0-m1.0": "/nix/store/nkq2y69kfbbjxrr9dvc385nskqs0cd67-replit-module-dotnet-7.0-m1.0", + "go-1.19-m1.0": "/nix/store/g1vrclpr65ynpldn7a6yvjsniaj3fb3r-replit-module-go-1.19-m1.0", + "haskell-9.0-m1.0": "/nix/store/7l7mafijx1lrxs85k6vzr3q5xzxi02fb-replit-module-haskell-9.0-m1.0", + "java-22.3-m1.0": "/nix/store/v09kxcfkm66bks2lxv5hknxh8i4ijzq9-replit-module-java-22.3-m1.0", + "lua-5.2-m1.0": "/nix/store/3g185mb4bzn2g9w0b7shw39x6ybby0g6-replit-module-lua-5.2-m1.0", + "nodejs-14.21-m1.1": "/nix/store/1igqvvk7agkxvz0ibi7vlsdkxnx5hnrj-replit-module-nodejs-14.21-m1.1", + "nodejs-16.18-m1.1": "/nix/store/751zzdn9cl7g4qx04k5szxm3jynfqa03-replit-module-nodejs-16.18-m1.1", + "nodejs-18.12-m1.1": "/nix/store/a1i4r09lc1qrnzcwv5bkfscc8nxk8b44-replit-module-nodejs-18.12-m1.1", + "nodejs-19.1-m1.1": "/nix/store/azs3v09dm03v27zrvhvhs7j1h5zm0y2s-replit-module-nodejs-19.1-m1.1", + "php-8.1-m1.0": "/nix/store/z9x4avlw2s0cmaqglbzb3ymb7cgv7hm4-replit-module-php-8.1-m1.0", + "python-3.10-m1.0": "/nix/store/ihcaap76i6xp3hzfayg6a0krx1pb5w52-replit-module-python-3.10-m1.0", + "qbasic-0.0-m1.1": "/nix/store/rpb9dg8c8cwiszfxa1xhw7z06yh59vn3-replit-module-qbasic-0.0-m1.1", + "r-4.2-m1.0": "/nix/store/c7wcs687dkxvfak0dr2gnb5ll69bv6yf-replit-module-r-4.2-m1.0", + "ruby-3.1-m1.0": "/nix/store/rxmyz9gv677v06jz64pqs7a9g2ppw0mj-replit-module-ruby-3.1-m1.0", + "rust-1.64-m1.0": "/nix/store/v2wq17m6chbxgv4vk2r4p4bqjp5r80vn-replit-module-rust-1.64-m1.0", + "swift-5.6-m1.0": "/nix/store/qp9bvj042a855xd4i0hrqbwz0p81zp4k-replit-module-swift-5.6-m1.0", + "web-3.0-m1.0": "/nix/store/37kafc7qxhji91175lgccmn2yx1dzw9m-replit-module-web-3.0-m1.0" +} +``` + +To build modules, you can do: + +``` +nix build .#bundle +``` + +which will create a `result` directory containing a symlink for each active module. + +## Lock Modules + +`lock_modules.py` is a script that generates a module registry file `modules.json`. +It should be run each time when before publishing a PR (but after committing your changes): + +``` +$ nix develop +$ python lock_modules.py +``` + +`modules.json` is similar to a lock file in used in common packagers in that it fixes +the exact version of each module. This file looks something like: + +```json +{ + "modules": { + "nodejs-18.12-m1.1": { + "commit": "4ec006c0eb247320e77c0abbf46b6f9e33370f81", + "created": "2023-05-04T16:52:42-04:00", + "path": "/nix/store/a1i4r09lc1qrnzcwv5bkfscc8nxk8b44-replit-module-nodejs-18.12-m1.1" + }, + "nodejs-19.1-m1.1": { + "commit": "4ec006c0eb247320e77c0abbf46b6f9e33370f81", + "created": "2023-05-04T16:52:42-04:00", + "path": "/nix/store/azs3v09dm03v27zrvhvhs7j1h5zm0y2s-replit-module-nodejs-19.1-m1.1" + }, + "go-1.19-m1.0": { + "commit": "4ec006c0eb247320e77c0abbf46b6f9e33370f81", + "created": "2023-05-04T16:52:42-04:00", + "path": "/nix/store/g1vrclpr65ynpldn7a6yvjsniaj3fb3r-replit-module-go-1.19-m1.0" + } + }, + "aliases": { + "nodejs": "nodejs-19.1-m1.1", + "nodejs-18.12": "nodejs-18.12-m1.1", + "nodejs-18.12-m1": "nodejs-18.12-m1.1", + "nodejs-19.1": "nodejs-19.1-m1.1", + "nodejs-19.1-m1": "nodejs-19.1-m1.1", + "go": "go-1.19-m1.0", + "go-1.19": "go-1.19-m1.0", + "go-1.19-m1": "go-1.19-m1.0" + } +} +``` + +The modules section is an append-only section. This means the contents of the value under a key, say `nodejs-19.1-m1.1` +cannot be changed. Each module contains: + +* commit - the git commit of the repo when `lock_modules.py` was ran. The script requires a clean working directory, +unless the `-d` flag is supplied +* created - the timestamp of the commit +* path - the output path of the module as returned by the `nix eval .#modules --json` command + +The aliases section points to the latest version of each module for a given shortened version specifier. + +If you made a modification in a module or a dependency of a module, re-running `lock_modules.py` will fail +an error like: +``` +Exception: go-1.19-m1.0 changed from /nix/store/g1vrclpr65ynpldn7a6yvjsniaj3fb3r-replit-module-go-1.19-m1.0 to /nix/store/pbrqcayg3ahawdld7j5kay97xli8zi0a-replit-module-go-1.19-m1.0 +``` + +To move forward, you'll have to increment the version of the associated module. + +See more about the versioning scheme at: https://replit.com/@util/Design-docs#goval/nixmodules_versions.md \ No newline at end of file diff --git a/flake.lock b/flake.lock index d4881f3c..05a66b96 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,20 @@ { "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "nixmodules-stable": { "inputs": { "nixpkgs": "nixpkgs" @@ -51,10 +66,47 @@ "type": "github" } }, + "nixpkgs_3": { + "locked": { + "lastModified": 1670276674, + "narHash": "sha256-FqZ7b2RpoHQ/jlG6JPcCNmG/DoUPCIvyaropUDFhF3Q=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "52e3e80afff4b16ccb7c52e9f0f5220552f03d04", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "prybar": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1683237785, + "narHash": "sha256-l4JoUTwISecR5DyBY3ShEvPxCV1F0l7FtH/bNWweUtY=", + "owner": "replit", + "repo": "prybar", + "rev": "65f486534054665f1b333689417c39acd370d3a5", + "type": "github" + }, + "original": { + "owner": "replit", + "repo": "prybar", + "rev": "65f486534054665f1b333689417c39acd370d3a5", + "type": "github" + } + }, "root": { "inputs": { "nixmodules-stable": "nixmodules-stable", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs_2", + "prybar": "prybar" } } }, diff --git a/flake.nix b/flake.nix index 7fdebd82..3c6997a1 100644 --- a/flake.nix +++ b/flake.nix @@ -2,13 +2,15 @@ description = "Nix expressions for defining Replit development environments"; inputs.nixpkgs.url = "github:nixos/nixpkgs?rev=52e3e80afff4b16ccb7c52e9f0f5220552f03d04"; inputs.nixmodules-stable.url = "github:replit/nixmodules?rev=d77c07009d6d0e09eaf7aa011cad530a644eca01"; + inputs.prybar.url = "github:replit/prybar?rev=65f486534054665f1b333689417c39acd370d3a5"; - outputs = { self, nixpkgs, nixmodules-stable, ... }: + outputs = { self, nixpkgs, nixmodules-stable, prybar, ... }: let mkPkgs = system: import nixpkgs { inherit system; - overlays = [ self.overlays.default ]; # ++ import ; + overlays = [ self.overlays.default prybar.overlays.default ]; # ++ import ; }; + pkgs = mkPkgs "x86_64-linux"; in { @@ -16,6 +18,16 @@ moduleit = self.packages.${prev.system}.moduleit; }; formatter.x86_64-linux = pkgs.nixpkgs-fmt; - packages.x86_64-linux = import ./pkgs { inherit pkgs self nixpkgs nixmodules-stable; }; + packages.x86_64-linux = import ./pkgs { + inherit pkgs self nixpkgs nixmodules-stable; + }; + devShells.x86_64-linux.default = pkgs.mkShell { + packages = [ + pkgs.python310 + ]; + }; + modules = import ./pkgs/modules { + inherit pkgs; + }; }; } diff --git a/lock_modules.py b/lock_modules.py new file mode 100644 index 00000000..8b04c69e --- /dev/null +++ b/lock_modules.py @@ -0,0 +1,136 @@ +import subprocess +import json +import os +import re +import argparse + +module_id_regex = re.compile(r'^([a-zA-Z0-9.]+)-([a-zA-Z0-9.]+)-m([0-9]+)\.([0-9]+)$') +module_registry_file = 'modules.json' + +def get_commit_info(): + output = subprocess.check_output(['git', 'show', '-s', '--format=format:%aI|%H']) + timestamp, commit = str(output, 'UTF-8').split('|') + return { + 'sha': commit, + 'timestamp': timestamp + } + +def is_working_directory_clean(): + output = subprocess.check_output(['git', 'status', '--porcelain', '--untracked-files=no']) + return len(output) == 0 + +def get_current_modules(): + output = subprocess.check_output(['nix', 'eval', '.#modules', '--json']) + return json.loads(output) + +def get_module_registry(): + if not os.path.isfile(module_registry_file): + return { 'modules': {}, 'aliases': {} } + f = open(module_registry_file, 'r') + registry = json.load(f) + f.close() + return registry + +def save_module_registry(registry): + f = open(module_registry_file, 'w') + json.dump(registry, f, indent = 2) + f.close() + print('Wrote %s' % module_registry_file) + +def parse_module_id(module_id): + match = module_id_regex.match(module_id) + id, community_version, major, minor = match.groups() + return { + 'id': id, + 'community_version': community_version, + 'major': major, + 'minor': minor, + } + +def is_version_greater(modinfo1, modinfo2): + if modinfo1['community_version'] == modinfo2['community_version']: + if modinfo1['major'] == modinfo2['major']: + return modinfo1['minor'] > modinfo2['minor'] + return modinfo1['major'] > modinfo2['major'] + return is_semver_greater(modinfo1['community_version'], modinfo2['community_version']) + +def is_semver_greater(semver1, semver2): + parts1 = list(map(int, semver1.split('.'))) + parts2 = list(map(int, semver2.split('.'))) + assert len(parts1) == len(parts2), "comparing semvars that have different number of parts: %s vs %s" % (semver1, semver2) + for i in range(len(parts1)): + if parts1[i] > parts2[i]: + return True + return False + +def generate_aliases(module_registry): + aliases = {} + for module_id in module_registry.keys(): + modinfo = parse_module_id(module_id) + short_alias = modinfo['id'] + medium_alias = "%s-%s" % (modinfo['id'], modinfo['community_version']) + long_alias = "%s-%s-m%s" % (modinfo['id'], modinfo['community_version'], modinfo['major']) + + if short_alias not in aliases or is_version_greater(modinfo, parse_module_id(aliases[short_alias])): + aliases[short_alias] = module_id + if medium_alias not in aliases or is_version_greater(modinfo, parse_module_id(aliases[medium_alias])): + aliases[medium_alias] = module_id + if long_alias not in aliases or is_version_greater(modinfo, parse_module_id(aliases[long_alias])): + aliases[long_alias] = module_id + return aliases + +def update_module_registry(module_registry): + commit = get_commit_info() + modules = get_current_modules() + changed = False + for module_id, module_path in modules.items(): + if module_id in module_registry: + # check for conflict + prev_path = module_registry[module_id]['path'] + if module_path != prev_path: + raise Exception('%s changed from %s to %s' % (module_id, prev_path, module_path)) + else: + print('%s unchanged' % module_id) + continue + + module_registry[module_id] = { + 'commit': commit['sha'], + 'created': commit['timestamp'], + 'path': module_path + } + print('%s added' % module_id) + changed = True + return changed + +def main(): + parser = argparse.ArgumentParser( + prog='lock_modules', + description='upserts current modules to %s' % module_registry_file, + ) + + parser.add_argument('-d', '--dirty', action='store_true') + parser.add_argument('-v', '--verify', action='store_true') + + args = parser.parse_args() + + if not args.dirty and not is_working_directory_clean(): + print('There are uncommitted changes. Exiting.') + exit(1) + + module_registry_top_level = get_module_registry() + module_registry = module_registry_top_level['modules'] + changed = update_module_registry(module_registry) + + if args.verify and changed: + print('%s is not up to date!!' % module_registry_file) + exit(1) + + if changed: + aliases = generate_aliases(module_registry) + save_module_registry({ + 'modules': module_registry, + 'aliases': aliases + }) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/pkgs/default.nix b/pkgs/default.nix index bfd629b1..6cfc1d86 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -3,31 +3,17 @@ with pkgs.lib; let - mkModule = path: pkgs.callPackage ./moduleit/entrypoint.nix { - configPath = path; - }; + modules = self.modules; revstring_long = self.rev or "dirty"; revstring = builtins.substring 0 7 revstring_long; - - modules = rec { - go = go_1; - go_1 = mkModule ./go; - - rust = rust_1; - rust_1 = mkModule ./rust; - - swift = swift_1; - swift_1 = mkModule ./swift; - }; - - modulesList = (mapAttrsToList (name: value: { inherit name; path = value;}) modules); - in rec { default = moduleit; moduleit = pkgs.callPackage ./moduleit { }; - bundle = pkgs.linkFarm "nixmodules-bundle-${revstring}" modulesList; + bundle = pkgs.linkFarm "nixmodules-bundle-${revstring}" ( + mapAttrsToList (name: value: { inherit name; path = value;}) modules + ); bundle-stable = nixmodules-stable.packages.${pkgs.system}.bundle; diff --git a/pkgs/moduleit/entrypoint.nix b/pkgs/moduleit/entrypoint.nix index f43a55e0..6c63107a 100644 --- a/pkgs/moduleit/entrypoint.nix +++ b/pkgs/moduleit/entrypoint.nix @@ -1,7 +1,6 @@ { pkgs ? import { } , configPath }: - (pkgs.lib.evalModules { modules = [ configPath diff --git a/pkgs/moduleit/module-definition.nix b/pkgs/moduleit/module-definition.nix index 8d60ce09..e0efa529 100644 --- a/pkgs/moduleit/module-definition.nix +++ b/pkgs/moduleit/module-definition.nix @@ -465,6 +465,11 @@ in { options = { + id = mkOption { + type = types.str; + description = "ID of the module"; + }; + name = mkOption { type = types.str; description = "Name of the module"; @@ -476,6 +481,11 @@ in default = ""; }; + community-version = mkOption { + type = types.str; + description = "The version of the language or framework provided by this module."; + }; + version = mkOption { type = types.str; description = "Version of the module"; @@ -575,8 +585,10 @@ in let moduleJSON = { + id = config.id; name = config.name; description = config.description; + community-version = config.community-version; version = config.version; env = { PATH = lib.makeBinPath config.packages; @@ -590,6 +602,6 @@ in }; in - pkgs.writeText "replit-module-${config.name}-${config.version}" (builtins.toJSON moduleJSON); + pkgs.writeText "replit-module-${config.id}-${config.community-version}-m${config.version}" (builtins.toJSON moduleJSON); }; } diff --git a/pkgs/modules/default.nix b/pkgs/modules/default.nix new file mode 100644 index 00000000..e699a565 --- /dev/null +++ b/pkgs/modules/default.nix @@ -0,0 +1,23 @@ +{ pkgs }: +let + mkModule = path: pkgs.callPackage ../moduleit/entrypoint.nix { + configPath = path; + }; + + modulesList = [ + (mkModule ./go) + (mkModule ./rust) + (mkModule ./swift) + ]; + + modules = builtins.listToAttrs ( + map (module: { name = get-module-id module; value = module; }) modulesList + ); + + get-module-id = module: + let + match = builtins.match "^\/nix\/store\/([^-]+)-replit-module-(.+)$" module.outPath; + in + builtins.elemAt match 1; +in + modules \ No newline at end of file diff --git a/pkgs/go/default.nix b/pkgs/modules/go/default.nix similarity index 76% rename from pkgs/go/default.nix rename to pkgs/modules/go/default.nix index 65ec7338..e51ac80c 100644 --- a/pkgs/go/default.nix +++ b/pkgs/modules/go/default.nix @@ -1,6 +1,11 @@ -{ pkgs, ... }: +{ pkgs, lib, ... }: +let + goversion = lib.versions.majorMinor pkgs.go.version; +in { - name = "GoTools"; + id = "go"; + name = "Go Tools"; + community-version = goversion; version = "1.0"; packages = with pkgs; [ diff --git a/pkgs/rust/default.nix b/pkgs/modules/rust/default.nix similarity index 88% rename from pkgs/rust/default.nix rename to pkgs/modules/rust/default.nix index 0347e0e2..d11f525f 100644 --- a/pkgs/rust/default.nix +++ b/pkgs/modules/rust/default.nix @@ -1,4 +1,4 @@ -{ pkgs, ... }: +{ pkgs, lib, ... }: let cargoRun = pkgs.writeScriptBin "cargo_run" '' if [ ! -f "$HOME/$REPL_SLUG/Cargo.toml" ]; then @@ -8,9 +8,12 @@ let ${pkgs.cargo}/bin/cargo run ''; + community-version = lib.versions.majorMinor pkgs.rustc.version; in { - name = "RustTools"; + id = "rust"; + name = "Rust Tools"; + inherit community-version; version = "1.0"; packages = with pkgs; [ diff --git a/pkgs/swift/default.nix b/pkgs/modules/swift/default.nix similarity index 84% rename from pkgs/swift/default.nix rename to pkgs/modules/swift/default.nix index 380cdb56..ea87b6dc 100644 --- a/pkgs/swift/default.nix +++ b/pkgs/modules/swift/default.nix @@ -1,5 +1,7 @@ -{ pkgs, ... }: +{ pkgs, lib, ... }: let + community-version = lib.versions.majorMinor pkgs.swift.version; + swiftc-wrapper = pkgs.stdenv.mkDerivation { name = "swiftc-wrapper"; buildInputs = [ pkgs.makeWrapper ]; @@ -14,7 +16,9 @@ let }; in { - name = "SwiftTools"; + id = "swift"; + name = "Swift Tools"; + inherit community-version; version = "1.0"; packages = with pkgs; [