Versioning poc

.gitignore

@@ -0,0 +1 @@
+result

README.md

@@ -0,0 +1,112 @@
+# Repl.it Nix Modules
+
+This repository holds Repl.it's Nix modules.
+
+* Each module is located as a folder under `pkgs/modules`
+* `pkgs/modules/default.nix` specifies a list of the active modules, some of which are parameterized with the version of the runtime or compiler
+
+To list all active modules, you can do:
+
+```
+nix eval .#modules --json | jq
+```
+Output might look like:
+```json
+{
+  "bun-0.5-m1.0": "/nix/store/nqwdhvs2n6fv82nkn77rg5wb3g1giwjs-replit-module-bun-0.5-m1.0",
+  "c-14.0-m1.0": "/nix/store/c8v74sbwivlj0mridwsdng883frwccy5-replit-module-c-14.0-m1.0",
+  "clojure-1.11-m1.0": "/nix/store/dsx4w6inr69f6c799qija5nf08q1ds39-replit-module-clojure-1.11-m1.0",
+  "cpp-14.0-m1.0": "/nix/store/065sbbghckzyirbxq239s4y3bcsr2wq2-replit-module-cpp-14.0-m1.0",
+  "dart-2.18-m1.0": "/nix/store/ampynbffmbjckc4xqzndlhbxncdljqfv-replit-module-dart-2.18-m1.0",
+  "dotnet-7.0-m1.0": "/nix/store/nkq2y69kfbbjxrr9dvc385nskqs0cd67-replit-module-dotnet-7.0-m1.0",
+  "go-1.19-m1.0": "/nix/store/g1vrclpr65ynpldn7a6yvjsniaj3fb3r-replit-module-go-1.19-m1.0",
+  "haskell-9.0-m1.0": "/nix/store/7l7mafijx1lrxs85k6vzr3q5xzxi02fb-replit-module-haskell-9.0-m1.0",
+  "java-22.3-m1.0": "/nix/store/v09kxcfkm66bks2lxv5hknxh8i4ijzq9-replit-module-java-22.3-m1.0",
+  "lua-5.2-m1.0": "/nix/store/3g185mb4bzn2g9w0b7shw39x6ybby0g6-replit-module-lua-5.2-m1.0",
+  "nodejs-14.21-m1.1": "/nix/store/1igqvvk7agkxvz0ibi7vlsdkxnx5hnrj-replit-module-nodejs-14.21-m1.1",
+  "nodejs-16.18-m1.1": "/nix/store/751zzdn9cl7g4qx04k5szxm3jynfqa03-replit-module-nodejs-16.18-m1.1",
+  "nodejs-18.12-m1.1": "/nix/store/a1i4r09lc1qrnzcwv5bkfscc8nxk8b44-replit-module-nodejs-18.12-m1.1",
+  "nodejs-19.1-m1.1": "/nix/store/azs3v09dm03v27zrvhvhs7j1h5zm0y2s-replit-module-nodejs-19.1-m1.1",
+  "php-8.1-m1.0": "/nix/store/z9x4avlw2s0cmaqglbzb3ymb7cgv7hm4-replit-module-php-8.1-m1.0",
+  "python-3.10-m1.0": "/nix/store/ihcaap76i6xp3hzfayg6a0krx1pb5w52-replit-module-python-3.10-m1.0",
+  "qbasic-0.0-m1.1": "/nix/store/rpb9dg8c8cwiszfxa1xhw7z06yh59vn3-replit-module-qbasic-0.0-m1.1",
+  "r-4.2-m1.0": "/nix/store/c7wcs687dkxvfak0dr2gnb5ll69bv6yf-replit-module-r-4.2-m1.0",
+  "ruby-3.1-m1.0": "/nix/store/rxmyz9gv677v06jz64pqs7a9g2ppw0mj-replit-module-ruby-3.1-m1.0",
+  "rust-1.64-m1.0": "/nix/store/v2wq17m6chbxgv4vk2r4p4bqjp5r80vn-replit-module-rust-1.64-m1.0",
+  "swift-5.6-m1.0": "/nix/store/qp9bvj042a855xd4i0hrqbwz0p81zp4k-replit-module-swift-5.6-m1.0",
+  "web-3.0-m1.0": "/nix/store/37kafc7qxhji91175lgccmn2yx1dzw9m-replit-module-web-3.0-m1.0"
+}
+```
+
+To build modules, you can do:
+
+```
+nix build .#bundle
+```
+
+which will create a `result` directory containing a symlink for each active module.
+
+## Lock Modules
+
+`lock_modules.py` is a script that generates a module registry file `modules.json`.
+It should be run each time when before publishing a PR (but after committing your changes):
+
+```
+$ nix develop
+$ python lock_modules.py
+```
+
+`modules.json` is similar to a lock file in used in common packagers in that it fixes
+the exact version of each module. This file looks something like:
+
+```json
+{
+  "modules": {
+    "nodejs-18.12-m1.1": {
+      "commit": "4ec006c0eb247320e77c0abbf46b6f9e33370f81",
+      "created": "2023-05-04T16:52:42-04:00",
+      "path": "/nix/store/a1i4r09lc1qrnzcwv5bkfscc8nxk8b44-replit-module-nodejs-18.12-m1.1"
+    },
+    "nodejs-19.1-m1.1": {
+      "commit": "4ec006c0eb247320e77c0abbf46b6f9e33370f81",
+      "created": "2023-05-04T16:52:42-04:00",
+      "path": "/nix/store/azs3v09dm03v27zrvhvhs7j1h5zm0y2s-replit-module-nodejs-19.1-m1.1"
+    },
+    "go-1.19-m1.0": {
+      "commit": "4ec006c0eb247320e77c0abbf46b6f9e33370f81",
+      "created": "2023-05-04T16:52:42-04:00",
+      "path": "/nix/store/g1vrclpr65ynpldn7a6yvjsniaj3fb3r-replit-module-go-1.19-m1.0"
+    }
+  },
+  "aliases": {
+    "nodejs": "nodejs-19.1-m1.1",
+    "nodejs-18.12": "nodejs-18.12-m1.1",
+    "nodejs-18.12-m1": "nodejs-18.12-m1.1",
+    "nodejs-19.1": "nodejs-19.1-m1.1",
+    "nodejs-19.1-m1": "nodejs-19.1-m1.1",
+    "go": "go-1.19-m1.0",
+    "go-1.19": "go-1.19-m1.0",
+    "go-1.19-m1": "go-1.19-m1.0"
+  }
+}
+```
+
+The modules section is an append-only section. This means the contents of the value under a key, say `nodejs-19.1-m1.1`
+cannot be changed. Each module contains:
+
+* commit - the git commit of the repo when `lock_modules.py` was ran. The script requires a clean working directory,
+unless the `-d` flag is supplied
+* created - the timestamp of the commit
+* path - the output path of the module as returned by the `nix eval .#modules --json` command
+
+The aliases section points to the latest version of each module for a given shortened version specifier.
+
+If you made a modification in a module or a dependency of a module, re-running `lock_modules.py` will fail
+an error like:
+```
+Exception: go-1.19-m1.0 changed from /nix/store/g1vrclpr65ynpldn7a6yvjsniaj3fb3r-replit-module-go-1.19-m1.0 to /nix/store/pbrqcayg3ahawdld7j5kay97xli8zi0a-replit-module-go-1.19-m1.0
+```
+
+To move forward, you'll have to increment the version of the associated module.
+
+See more about the versioning scheme at: https://replit.com/@util/Design-docs#goval/nixmodules_versions.md

flake.nix

@@ -2,20 +2,32 @@
   description = "Nix expressions for defining Replit development environments";
   inputs.nixpkgs.url = "github:nixos/nixpkgs?rev=52e3e80afff4b16ccb7c52e9f0f5220552f03d04";
   inputs.nixmodules-stable.url = "github:replit/nixmodules?rev=d77c07009d6d0e09eaf7aa011cad530a644eca01";
+  inputs.prybar.url = "github:replit/prybar?rev=65f486534054665f1b333689417c39acd370d3a5";
 
-  outputs = { self, nixpkgs, nixmodules-stable, ... }:
+  outputs = { self, nixpkgs, nixmodules-stable, prybar, ... }:
     let
       mkPkgs = system: import nixpkgs {
         inherit system;
-        overlays = [ self.overlays.default ]; # ++ import ;
+        overlays = [ self.overlays.default prybar.overlays.default ]; # ++ import ;
       };
+
       pkgs = mkPkgs "x86_64-linux";
     in
     {
       overlays.default = final: prev: {
         moduleit = self.packages.${prev.system}.moduleit;
       };
       formatter.x86_64-linux = pkgs.nixpkgs-fmt;
-      packages.x86_64-linux = import ./pkgs { inherit pkgs self nixpkgs nixmodules-stable; };
+      packages.x86_64-linux = import ./pkgs {
+        inherit pkgs self nixpkgs nixmodules-stable; 
+      };
+      devShells.x86_64-linux.default = pkgs.mkShell {
+        packages = [
+          pkgs.python310
+        ];
+      };
+      modules = import ./pkgs/modules {
+        inherit pkgs;
+      };
     };
 }

lock_modules.py

@@ -0,0 +1,136 @@
+import subprocess
+import json
+import os
+import re
+import argparse
+
+module_id_regex = re.compile(r'^([a-zA-Z0-9.]+)-([a-zA-Z0-9.]+)-m([0-9]+)\.([0-9]+)$')
+module_registry_file = 'modules.json'
+
+def get_commit_info():
+  output = subprocess.check_output(['git', 'show', '-s', '--format=format:%aI|%H'])
+  timestamp, commit = str(output, 'UTF-8').split('|')
+  return {
+    'sha': commit,
+    'timestamp': timestamp
+  }
+
+def is_working_directory_clean():
+  output = subprocess.check_output(['git', 'status', '--porcelain', '--untracked-files=no'])
+  return len(output) == 0
+
+def get_current_modules():
+  output = subprocess.check_output(['nix', 'eval', '.#modules', '--json'])
+  return json.loads(output)
+
+def get_module_registry():
+  if not os.path.isfile(module_registry_file):
+    return { 'modules': {}, 'aliases': {} }
+  f = open(module_registry_file, 'r')
+  registry = json.load(f)
+  f.close()
+  return registry
+
+def save_module_registry(registry):
+  f = open(module_registry_file, 'w')
+  json.dump(registry, f, indent = 2)
+  f.close()
+  print('Wrote %s' % module_registry_file)
+
+def parse_module_id(module_id):
+  match = module_id_regex.match(module_id)
+  id, community_version, major, minor = match.groups()
+  return {
+    'id': id,
+    'community_version': community_version,
+    'major': major,
+    'minor': minor,
+  }
+
+def is_version_greater(modinfo1, modinfo2):
+  if modinfo1['community_version'] == modinfo2['community_version']:
+    if modinfo1['major'] == modinfo2['major']:
+      return modinfo1['minor'] > modinfo2['minor']
+    return modinfo1['major'] > modinfo2['major']
+  return is_semver_greater(modinfo1['community_version'], modinfo2['community_version'])
+
+def is_semver_greater(semver1, semver2):
+  parts1 = list(map(int, semver1.split('.')))
+  parts2 = list(map(int, semver2.split('.')))
+  assert len(parts1) == len(parts2), "comparing semvars that have different number of parts: %s vs %s" % (semver1, semver2)
+  for i in range(len(parts1)):
+    if parts1[i] > parts2[i]:
+      return True
+  return False
+
+def generate_aliases(module_registry):
+  aliases = {}
+  for module_id in module_registry.keys():
+    modinfo = parse_module_id(module_id)
+    short_alias = modinfo['id']
+    medium_alias = "%s-%s" % (modinfo['id'], modinfo['community_version'])
+    long_alias = "%s-%s-m%s" % (modinfo['id'], modinfo['community_version'], modinfo['major'])
+
+    if short_alias not in aliases or is_version_greater(modinfo, parse_module_id(aliases[short_alias])):
+      aliases[short_alias] = module_id
+    if medium_alias not in aliases or is_version_greater(modinfo, parse_module_id(aliases[medium_alias])):
+      aliases[medium_alias] = module_id
+    if long_alias not in aliases or is_version_greater(modinfo, parse_module_id(aliases[long_alias])):
+      aliases[long_alias] = module_id
+  return aliases
+
+def update_module_registry(module_registry):
+  commit = get_commit_info()
+  modules = get_current_modules()
+  changed = False
+  for module_id, module_path in modules.items():
+    if module_id in module_registry:
+      # check for conflict
+      prev_path = module_registry[module_id]['path']
+      if module_path != prev_path:
+        raise Exception('%s changed from %s to %s' % (module_id, prev_path, module_path))
+      else:
+        print('%s unchanged' % module_id)
+        continue
+
+    module_registry[module_id] = {
+      'commit': commit['sha'],
+      'created': commit['timestamp'],
+      'path': module_path
+    }
+    print('%s added' % module_id)
+    changed = True
+  return changed
+
+def main():
+  parser = argparse.ArgumentParser(
+    prog='lock_modules',
+    description='upserts current modules to %s' % module_registry_file,
+  )
+
+  parser.add_argument('-d', '--dirty', action='store_true')
+  parser.add_argument('-v', '--verify', action='store_true')
+
+  args = parser.parse_args()
+
+  if not args.dirty and not is_working_directory_clean():
+    print('There are uncommitted changes. Exiting.')
+    exit(1)
+
+  module_registry_top_level = get_module_registry()
+  module_registry = module_registry_top_level['modules']
+  changed = update_module_registry(module_registry)
+
+  if args.verify and changed:
+    print('%s is not up to date!!' % module_registry_file)
+    exit(1)
+
+  if changed:
+    aliases = generate_aliases(module_registry)
+    save_module_registry({
+      'modules': module_registry,
+      'aliases': aliases
+    })
+
+if __name__ == '__main__':
+  main()

pkgs/default.nix

@@ -3,31 +3,17 @@
 with pkgs.lib;
 
 let
-  mkModule = path: pkgs.callPackage ./moduleit/entrypoint.nix {
-    configPath = path;
-  };
+  modules = self.modules;
   revstring_long = self.rev or "dirty";
   revstring = builtins.substring 0 7 revstring_long;
-
-  modules = rec {
-    go = go_1;
-    go_1 = mkModule ./go;
-
-    rust = rust_1;
-    rust_1 = mkModule ./rust;
-
-    swift = swift_1;
-    swift_1 = mkModule ./swift;
-  };
-
-  modulesList = (mapAttrsToList (name: value: { inherit name; path = value;}) modules);
-
 in
 rec {
   default = moduleit;
   moduleit = pkgs.callPackage ./moduleit { };
 
-  bundle = pkgs.linkFarm "nixmodules-bundle-${revstring}" modulesList;
+  bundle = pkgs.linkFarm "nixmodules-bundle-${revstring}" (
+    mapAttrsToList (name: value: { inherit name; path = value;}) modules
+  );
 
   bundle-stable = nixmodules-stable.packages.${pkgs.system}.bundle;