kookie
/
nomicon
1
0
Fork 0
Code Issues Projects Releases 1 Activity

update-luarocks-packages: use pluginupdate.py

Browse Source 
Cleans up the common interface between the updaters.
Restores the ability to regen the lua packages in parallel.
launchpad/nixpkgs/master
Matthieu Coudron 3 years ago
parent 4b88ab8c5c
commit 8ddad244ea
5 changed files with 238 additions and 222 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 111
      maintainers/scripts/pluginupdate.py
  2. 149
      maintainers/scripts/update-luarocks-packages
  3. 79
      pkgs/applications/editors/kakoune/plugins/update.py
  4. 35
      pkgs/development/lua-modules/generated-packages.nix
  5. 86
      pkgs/misc/vim-plugins/update.py

111
maintainers/scripts/pluginupdate.py
Unescape View File

@ -42,8 +42,6 @@ LOG_LEVELS = {
}
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
"""Retry calling the decorated function using an exponential backoff.
@ -203,7 +201,6 @@ class Editor:
name: str,
root: Path,
get_plugins: str,
generate_nix: Callable[[List[Tuple[str, str, Plugin]], str], None],
default_in: Optional[Path] = None,
default_out: Optional[Path] = None,
deprecated: Optional[Path] = None,
@ -213,7 +210,6 @@ class Editor:
self.name = name
self.root = root
self.get_plugins = get_plugins
self._generate_nix = generate_nix
self.default_in = default_in or root.joinpath(f"{name}-plugin-names")
self.default_out = default_out or root.joinpath("generated.nix")
self.deprecated = deprecated or root.joinpath("deprecated.json")
@ -226,9 +222,9 @@ class Editor:
def load_plugin_spec(self, plugin_file) -> List[PluginDesc]:
return load_plugin_spec(plugin_file)
def generate_nix(self, plugins, outfile):
def generate_nix(self, plugins, outfile: str):
'''Returns nothing for now, writes directly to outfile'''
self._generate_nix(plugins, outfile)
raise NotImplementedError()
def get_update(self, input_file: str, outfile: str, proc: int):
return get_update(input_file, outfile, proc, editor=self)
@ -237,9 +233,58 @@ class Editor:
def attr_path(self):
return self.name + "Plugins"
def get_drv_name(self, name: str):
return self.attr_path + "." + name
def rewrite_input(self, *args, **kwargs):
return rewrite_input(*args, **kwargs)
def create_parser(self):
parser = argparse.ArgumentParser(
description=(
f"Updates nix derivations for {self.name} plugins"
f"By default from {self.default_in} to {self.default_out}"
)
)
parser.add_argument(
"--add",
dest="add_plugins",
default=[],
action="append",
help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo",
)
parser.add_argument(
"--input-names",
"-i",
dest="input_file",
default=self.default_in,
help="A list of plugins in the form owner/repo",
)
parser.add_argument(
"--out",
"-o",
dest="outfile",
default=self.default_out,
help="Filename to save generated nix code",
)
parser.add_argument(
"--proc",
"-p",
dest="proc",
type=int,
default=30,
help="Number of concurrent processes to spawn.",
)
parser.add_argument(
"--no-commit", "-n", action="store_true", default=False,
help="Whether to autocommit changes"
)
parser.add_argument(
"--debug", "-d", choices=LOG_LEVELS.keys(),
default=logging.getLevelName(logging.WARN),
help="Adjust log level"
)
return parser
@ -466,54 +511,6 @@ def rewrite_input(
with open(input_file, "w") as f:
f.writelines(lines)
# TODO move to Editor ?
def parse_args(editor: Editor):
parser = argparse.ArgumentParser(
description=(
f"Updates nix derivations for {editor.name} plugins"
f"By default from {editor.default_in} to {editor.default_out}"
)
)
parser.add_argument(
"--add",
dest="add_plugins",
default=[],
action="append",
help=f"Plugin to add to {editor.attr_path} from Github in the form owner/repo",
)
parser.add_argument(
"--input-names",
"-i",
dest="input_file",
default=editor.default_in,
help="A list of plugins in the form owner/repo",
)
parser.add_argument(
"--out",
"-o",
dest="outfile",
default=editor.default_out,
help="Filename to save generated nix code",
)
parser.add_argument(
"--proc",
"-p",
dest="proc",
type=int,
default=30,
help="Number of concurrent processes to spawn.",
)
parser.add_argument(
"--no-commit", "-n", action="store_true", default=False,
help="Whether to autocommit changes"
)
parser.add_argument(
"--debug", "-d", choices=LOG_LEVELS.keys(),
default=logging.getLevelName(logging.WARN),
help="Adjust log level"
)
return parser.parse_args()
def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
repo.index.add([str(f.resolve()) for f in files])
@ -547,12 +544,10 @@ def get_update(input_file: str, outfile: str, proc: int, editor: Editor):
return update
def update_plugins(editor: Editor):
def update_plugins(editor: Editor, args):
"""The main entry function of this module. All input arguments are grouped in the `Editor`."""
args = parse_args(editor)
log.setLevel(LOG_LEVELS[args.debug])
log.info("Start updating plugins")
nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True)
update = editor.get_update(args.input_file, args.outfile, args.proc)
@ -581,7 +576,7 @@ def update_plugins(editor: Editor):
if autocommit:
commit(
nixpkgs_repo,
"{editor.attr_path}.{name}: init at {version}".format(
"{editor.get_drv_name name}: init at {version}".format(
editor=editor.name, name=plugin.normalized_name, version=plugin.version
),
[args.outfile, args.input_file],

149
maintainers/scripts/update-luarocks-packages
Unescape View File

@ -16,20 +16,17 @@ from dataclasses import dataclass
import subprocess
import csv
import logging
import textwrap
from multiprocessing.dummy import Pool
from typing import List
from typing import List, Tuple
from pathlib import Path
LOG_LEVELS = {
logging.getLevelName(level): level for level in [
logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR ]
}
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))).parent.parent
from pluginupdate import Editor, parse_args, update_plugins, PluginDesc, CleanEnvironment
from pluginupdate import Editor, update_plugins, PluginDesc, CleanEnvironment, LOG_LEVELS, Cache
PKG_LIST="maintainers/scripts/luarocks-packages.csv"
TMP_FILE="$(mktemp)"
@ -67,12 +64,11 @@ class LuaEditor(Editor):
def get_current_plugins(self):
return []
def load_plugin_spec(self, input_file) -> List[PluginDesc]:
def load_plugin_spec(self, input_file) -> List[LuaPlugin]:
luaPackages = []
csvfilename=input_file
log.info("Loading package descriptions from %s", csvfilename)
with open(csvfilename, newline='') as csvfile:
reader = csv.DictReader(csvfile,)
for row in reader:
@ -81,96 +77,115 @@ class LuaEditor(Editor):
luaPackages.append(plugin)
return luaPackages
def generate_nix(
self,
results: List[Tuple[LuaPlugin, str]],
outfilename: str
):
with tempfile.NamedTemporaryFile("w+") as f:
f.write(HEADER)
header2 = textwrap.dedent(
# header2 = inspect.cleandoc(
"""
{ self, stdenv, lib, fetchurl, fetchgit, ... } @ args:
self: super:
with self;
{
""")
f.write(header2)
for (plugin, nix_expr) in results:
f.write(f"{plugin.normalized_name} = {nix_expr}")
f.write(FOOTER)
f.flush()
# if everything went fine, move the generated file to its destination
# using copy since move doesn't work across disks
shutil.copy(f.name, outfilename)
print(f"updated {outfilename}")
@property
def attr_path(self):
return "luaPackages"
def get_update(self, input_file: str, outfile: str, _: int):
def get_update(self, input_file: str, outfile: str, proc: int):
cache: Cache = Cache(self.cache_file)
_prefetch = generate_pkg_nix
def update() -> dict:
plugin_specs = self.load_plugin_spec(input_file)
sorted_plugin_specs = sorted(plugin_specs, key=lambda v: v.name.lower())
try:
pool = Pool(processes=proc)
results = pool.map(_prefetch, sorted_plugin_specs)
finally:
pass
self.generate_nix(plugin_specs, outfile)
self.generate_nix(results, outfile)
redirects = []
return redirects
return update
def rewrite_input(self, *args, **kwargs):
def rewrite_input(self, input_file: str, *args, **kwargs):
# vim plugin reads the file before update but that shouldn't be our case
# not implemented yet
# fieldnames = ['name', 'server', 'version', 'luaversion', 'maintainers']
# input_file = "toto.csv"
# with open(input_file, newline='') as csvfile:
# writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
# writer.writeheader()
# for row in reader:
# # name,server,version,luaversion,maintainers
# plugin = LuaPlugin(**row)
# luaPackages.append(plugin)
pass
def generate_nix(
plugins: List[LuaPlugin],
outfilename: str
):
sorted_plugins = sorted(plugins, key=lambda v: v.name.lower())
# plug = {}
# selon le manifest luarocks.org/manifest
def _generate_pkg_nix(plug):
cmd = [ "luarocks", "nix", plug.name]
if plug.server:
cmd.append(f"--only-server={plug.server}")
if plug.maintainers:
cmd.append(f"--maintainers={plug.maintainers}")
if plug.version:
cmd.append(plug.version)
if plug.luaversion:
with CleanEnvironment():
local_pkgs = str(ROOT.resolve())
cmd2 = ["nix-build", "--no-out-link", local_pkgs, "-A", f"{plug.luaversion}"]
log.debug("running %s", cmd2)
lua_drv_path=subprocess.check_output(cmd2, text=True).strip()
cmd.append(f"--lua-dir={lua_drv_path}/bin")
log.debug("running %s", cmd)
output = subprocess.check_output(cmd, text=True)
return output
with tempfile.NamedTemporaryFile("w+") as f:
f.write(HEADER)
f.write("""
{ self, stdenv, lib, fetchurl, fetchgit, ... } @ args:
self: super:
with self;
{
""")
def generate_pkg_nix(plug: LuaPlugin):
'''
Generate nix expression for a luarocks package
Our cache key associates "p.name-p.version" to its rockspec
'''
log.debug("Generating nix expression for %s", plug.name)
cmd = [ "luarocks", "nix", plug.name]
for plugin in sorted_plugins:
if plug.server:
cmd.append(f"--only-server={plug.server}")
nix_expr = _generate_pkg_nix(plugin)
f.write(f"{plugin.normalized_name} = {nix_expr}"
)
f.write(FOOTER)
f.flush()
if plug.maintainers:
cmd.append(f"--maintainers={plug.maintainers}")
# if everything went fine, move the generated file to its destination
# using copy since move doesn't work across disks
shutil.copy(f.name, outfilename)
if plug.version:
cmd.append(plug.version)
print(f"updated {outfilename}")
if plug.luaversion:
with CleanEnvironment():
local_pkgs = str(ROOT.resolve())
cmd2 = ["nix-build", "--no-out-link", local_pkgs, "-A", f"{plug.luaversion}"]
def load_plugin_spec():
pass
log.debug("running %s", ' '.join(cmd2))
lua_drv_path=subprocess.check_output(cmd2, text=True).strip()
cmd.append(f"--lua-dir={lua_drv_path}/bin")
log.debug("running %s", cmd)
output = subprocess.check_output(cmd, text=True)
return (plug, output)
def main():
editor = LuaEditor("lua", ROOT, '', generate_nix,
editor = LuaEditor("lua", ROOT, '',
default_in = ROOT.joinpath(PKG_LIST),
default_out = ROOT.joinpath(GENERATED_NIXFILE)
)
args = parse_args(editor)
parser = editor.create_parser()
args = parser.parse_args()
log.setLevel(LOG_LEVELS[args.debug])
update_plugins(editor)
update_plugins(editor, args)
if __name__ == "__main__":

79
pkgs/applications/editors/kakoune/plugins/update.py
Unescape View File

@ -39,52 +39,57 @@ in lib.filterAttrs (n: v: v != null) checksums)"""
HEADER = "# This file has been generated by ./pkgs/applications/editors/kakoune/plugins/update.py. Do not edit!"
class KakouneEditor(pluginupdate.Editor):
def generate_nix(plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str):
sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower())
with open(outfile, "w+") as f:
f.write(HEADER)
f.write(
"""
{ lib, buildKakounePluginFrom2Nix, fetchFromGitHub, overrides ? (self: super: {}) }:
let
packages = ( self:
{"""
)
for owner, repo, plugin in sorted_plugins:
if plugin.has_submodules:
submodule_attr = "\n fetchSubmodules = true;"
else:
submodule_attr = ""
def generate_nix(plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str):
sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower())
with open(outfile, "w+") as f:
f.write(HEADER)
f.write(
f"""
{plugin.normalized_name} = buildKakounePluginFrom2Nix {{
pname = "{plugin.normalized_name}";
version = "{plugin.version}";
src = fetchFromGitHub {{
owner = "{owner}";
repo = "{repo}";
rev = "{plugin.commit}";
sha256 = "{plugin.sha256}";{submodule_attr}
"""
{ lib, buildKakounePluginFrom2Nix, fetchFromGitHub, overrides ? (self: super: {}) }:
let
packages = ( self:
{"""
)
for owner, repo, plugin in sorted_plugins:
if plugin.has_submodules:
submodule_attr = "\n fetchSubmodules = true;"
else:
submodule_attr = ""
f.write(
f"""
{plugin.normalized_name} = buildKakounePluginFrom2Nix {{
pname = "{plugin.normalized_name}";
version = "{plugin.version}";
src = fetchFromGitHub {{
owner = "{owner}";
repo = "{repo}";
rev = "{plugin.commit}";
sha256 = "{plugin.sha256}";{submodule_attr}
}};
meta.homepage = "https://github.com/{owner}/{repo}/";
}};
meta.homepage = "https://github.com/{owner}/{repo}/";
}};
"""
"""
)
f.write(
"""
});
in lib.fix' (lib.extends overrides packages)
"""
)
f.write(
"""
});
in lib.fix' (lib.extends overrides packages)
"""
)
print(f"updated {outfile}")
print(f"updated {outfile}")
def main():
editor = pluginupdate.Editor("kakoune", ROOT, GET_PLUGINS, generate_nix)
pluginupdate.update_plugins(editor)
editor = KakouneEditor("kakoune", ROOT, GET_PLUGINS)
parser = editor.create_parser()
args = parser.parse_args()
pluginupdate.update_plugins(editor, args)
if __name__ == "__main__":

35
pkgs/development/lua-modules/generated-packages.nix
Unescape View File

@ -110,20 +110,20 @@ binaryheap = buildLuarocksPackage {
bit32 = buildLuarocksPackage {
pname = "bit32";
version = "5.3.0-1";
version = "5.3.5.1-1";
src = fetchurl {
url = "https://luarocks.org/bit32-5.3.0-1.src.rock";
sha256 = "19i7kc2pfg9hc6qjq4kka43q6qk71bkl2rzvrjaks6283q6wfyzy";
};
disabled = (luaOlder "5.1");
disabled = (luaOlder "5.1") || (luaAtLeast "5.5");
propagatedBuildInputs = [ lua ];
meta = with lib; {
homepage = "http://www.lua.org/manual/5.2/manual.html#6.7";
description = "Lua 5.2 bit manipulation library";
maintainers = with maintainers; [ lblasc ];
license.fullName = "MIT/X11";
license.fullName = "MIT";
};
};
@ -169,16 +169,15 @@ cassowary = buildLuarocksPackage {
license.fullName = "Apache 2";
};
};
compat53 = buildLuarocksPackage {
pname = "compat53";
version = "0.7-1";
version = "0.8-1";
src = fetchurl {
url = "https://luarocks.org/compat53-0.7-1.src.rock";
sha256 = "0kpaxbpgrwjn4jjlb17fn29a09w6lw732d21bi0302kqcaixqpyb";
};
disabled = (luaOlder "5.1") || (luaAtLeast "5.4");
disabled = (luaOlder "5.1") || (luaAtLeast "5.5");
propagatedBuildInputs = [ lua ];
meta = with lib; {
@ -275,7 +274,7 @@ cyrussasl = buildLuarocksPackage {
digestif = buildLuarocksPackage {
pname = "digestif";
version = "0.2-1";
version = "0.4-1";
src = fetchurl {
url = "mirror://luarocks/digestif-0.2-1.src.rock";
@ -328,7 +327,7 @@ fifo = buildLuarocksPackage {
http = buildLuarocksPackage {
pname = "http";
version = "0.3-0";
version = "0.4-0";
src = fetchurl {
url = "https://luarocks.org/http-0.3-0.src.rock";
@ -1130,13 +1129,13 @@ luaevent = buildLuarocksPackage {
luaexpat = buildLuarocksPackage {
pname = "luaexpat";
version = "1.3.0-1";
version = "1.3.3-1";
src = fetchurl {
url = "https://luarocks.org/luaexpat-1.3.0-1.src.rock";
sha256 = "15jqz5q12i9zvjyagzwz2lrpzya64mih8v1hxwr0wl2gsjh86y5a";
};
disabled = (luaOlder "5.1");
disabled = (luaOlder "5.0");
propagatedBuildInputs = [ lua ];
meta = with lib; {
@ -1167,7 +1166,7 @@ luaffi = buildLuarocksPackage {
luafilesystem = buildLuarocksPackage {
pname = "luafilesystem";
version = "1.7.0-2";
version = "1.8.0-1";
src = fetchurl {
url = "https://luarocks.org/luafilesystem-1.7.0-2.src.rock";
@ -1220,14 +1219,14 @@ luaossl = buildLuarocksPackage {
luaposix = buildLuarocksPackage {
pname = "luaposix";
version = "34.1.1-1";
version = "35.0-1";
src = fetchurl {
url = "https://luarocks.org/luaposix-34.1.1-1.src.rock";
sha256 = "1l9pkn3g0nzlbmmfj12rhfwvkqb06c21ydqxqgmnmd3w9z4ck53w";
};
disabled = (luaOlder "5.1") || (luaAtLeast "5.4");
propagatedBuildInputs = [ bit32 lua ];
disabled = (luaOlder "5.1") || (luaAtLeast "5.5");
propagatedBuildInputs = [ lua ];
meta = with lib; {
homepage = "http://github.com/luaposix/luaposix/";
@ -1408,7 +1407,6 @@ luazip = buildLuarocksPackage {
license.fullName = "MIT";
};
};
luuid = buildLuarocksPackage {
pname = "luuid";
version = "20120509-2";
@ -1429,11 +1427,11 @@ luuid = buildLuarocksPackage {
luv = buildLuarocksPackage {
pname = "luv";
version = "1.30.0-0";
version = "1.41.1-0";
src = fetchurl {
url = "https://luarocks.org/luv-1.30.0-0.src.rock";
sha256 = "1z5sdq9ld4sm5pws9qxpk9cadv9i7ycwad1zwsa57pj67gly11vi";
url = "https://luarocks.org/luv-1.41.1-0.src.rock";
sha256 = "0l1v07nhrkzsddbcc4bak382b5flyw6x8g4i394ylbfl25zwcmai";
};
disabled = (luaOlder "5.1");
propagatedBuildInputs = [ lua ];
@ -1546,7 +1544,6 @@ mpack = buildLuarocksPackage {
license.fullName = "MIT";
};
};
nvim-client = buildLuarocksPackage {
pname = "nvim-client";
version = "0.2.2-1";

86
pkgs/misc/vim-plugins/update.py
Unescape View File

@ -11,9 +11,14 @@
import inspect
import os
import sys
import logging
import textwrap
from typing import List, Tuple
from pathlib import Path
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
# Import plugin update library from maintainers/scripts/pluginupdate.py
ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))))
sys.path.insert(0, os.path.join(ROOT.parent.parent.parent, "maintainers", "scripts"))
@ -40,50 +45,49 @@ HEADER = (
)
def generate_nix(plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str):
sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower())
with open(outfile, "w+") as f:
f.write(HEADER)
f.write(
"""
{ lib, buildVimPluginFrom2Nix, fetchFromGitHub }:
final: prev:
{"""
)
for owner, repo, plugin in sorted_plugins:
if plugin.has_submodules:
submodule_attr = "\n fetchSubmodules = true;"
else:
submodule_attr = ""
f.write(
f"""
{plugin.normalized_name} = buildVimPluginFrom2Nix {{
pname = "{plugin.normalized_name}";
version = "{plugin.version}";
src = fetchFromGitHub {{
owner = "{owner}";
repo = "{repo}";
rev = "{plugin.commit}";
sha256 = "{plugin.sha256}";{submodule_attr}
}};
meta.homepage = "https://github.com/{owner}/{repo}/";
}};
"""
)
f.write(
"""
}
"""
)
print(f"updated {outfile}")
class VimEditor(pluginupdate.Editor):
def generate_nix(self, plugins: List[Tuple[str, str, pluginupdate.Plugin]], outfile: str):
sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower())
with open(outfile, "w+") as f:
f.write(HEADER)
f.write(textwrap.dedent("""
{ lib, buildVimPluginFrom2Nix, fetchFromGitHub }:
final: prev:
{"""
))
for owner, repo, plugin in sorted_plugins:
if plugin.has_submodules:
submodule_attr = "\n fetchSubmodules = true;"
else:
submodule_attr = ""
f.write(textwrap.indent(textwrap.dedent(
f"""
{plugin.normalized_name} = buildVimPluginFrom2Nix {{
pname = "{plugin.normalized_name}";
version = "{plugin.version}";
src = fetchFromGitHub {{
owner = "{owner}";
repo = "{repo}";
rev = "{plugin.commit}";
sha256 = "{plugin.sha256}";{submodule_attr}
}};
meta.homepage = "https://github.com/{owner}/{repo}/";
}};
"""
), ' '))
f.write("\n}")
print(f"updated {outfile}")
def main():
editor = pluginupdate.Editor("vim", ROOT, GET_PLUGINS, generate_nix)
pluginupdate.update_plugins(editor)
editor = VimEditor("vim", ROOT, GET_PLUGINS)
parser = editor.create_parser()
args = parser.parse_args()
pluginupdate.update_plugins(editor, args)
if __name__ == "__main__":

Write Preview
Loading…
Cancel
Save