resholve: init at 0.4.0 (#85827)
resholve: init at 0.4.0 resholve attempts to resolve executables in shell scripts. Includes Nix builder for resolving dependencies in Nix-built shell projects.wip/yesman
parent
645f39f33e
commit
6fd9283bba
@ -0,0 +1,138 @@ |
||||
# Using resholve's Nix API |
||||
|
||||
resholve converts bare executable references in shell scripts to absolute |
||||
paths. This will hopefully make its way into the Nixpkgs manual soon, but |
||||
until then I'll outline how to use the `resholvePackage` function. |
||||
|
||||
> Fair warning: resholve does *not* aspire to resolving all valid Shell |
||||
> scripts. It depends on the OSH/Oil parser, which aims to support most (but |
||||
> not all) Bash, and aims to be a ~90% sort of solution. |
||||
|
||||
Let's start with a simple example from one of my own projects: |
||||
|
||||
```nix |
||||
{ stdenv, lib, resholvePackage, fetchFromGitHub, bashup-events44, bashInteractive_5, doCheck ? true, shellcheck }: |
||||
|
||||
resholvePackage rec { |
||||
pname = "shellswain"; |
||||
version = "unreleased"; |
||||
|
||||
src = fetchFromGitHub { |
||||
# ... |
||||
}; |
||||
|
||||
solutions = { |
||||
profile = { |
||||
# the only *required* arguments |
||||
scripts = [ "bin/shellswain.bash" ]; |
||||
interpreter = "none"; |
||||
inputs = [ bashup-events44 ]; |
||||
}; |
||||
}; |
||||
|
||||
makeFlags = [ "prefix=${placeholder "out"}" ]; |
||||
|
||||
inherit doCheck; |
||||
checkInputs = [ shellcheck ]; |
||||
|
||||
# ... |
||||
} |
||||
``` |
||||
|
||||
I'll focus on the `solutions` attribute, since this is the only part |
||||
that differs from other derivations. |
||||
|
||||
Each "solution" (k=v pair) |
||||
describes one resholve invocation. For most shell packages, one |
||||
invocation will probably be enough. resholve will make you be very |
||||
explicit about your script's dependencies, and it may also need your |
||||
help sorting out some references or problems that it can't safely |
||||
handle on its own. |
||||
|
||||
If you have more than one script, and your scripts need conflicting |
||||
directives, you can specify more than one solution to resolve the |
||||
scripts separately, but still produce a single package. |
||||
|
||||
Let's take a closer look: |
||||
|
||||
```nix |
||||
solutions = { |
||||
# each solution has a short name; this is what you'd use to |
||||
# override the settings of this solution, and it may also show up |
||||
# in (some) error messages. |
||||
profile = { |
||||
# specify one or more $out-relative script paths (unlike many |
||||
# builders, resholve will modify the output files during fixup |
||||
# to correctly resolve scripts that source within the package) |
||||
scripts = [ "bin/shellswain.bash" ]; |
||||
# "none" for no shebang, "${bash}/bin/bash" for bash, etc. |
||||
interpreter = "none"; |
||||
# packages resholve should resolve executables from |
||||
inputs = [ bashup-events44 ]; |
||||
}; |
||||
}; |
||||
``` |
||||
|
||||
resholve has a (growing) number of options for handling more complex |
||||
scripts. I won't cover these in excruciating detail here. You can find |
||||
more information about these in `man resholve` via `nixpkgs.resholve`. |
||||
|
||||
Instead, we'll look at the general form of the solutions attrset: |
||||
|
||||
```nix |
||||
solutions = { |
||||
shortname = { |
||||
# required |
||||
# $out-relative paths to try resolving |
||||
scripts = [ "bin/shunit2" ]; |
||||
# packages to resolve executables from |
||||
inputs = [ coreutils gnused gnugrep findutils ]; |
||||
# path for shebang, or 'none' to omit shebang |
||||
interpreter = "${bash}/bin/bash"; |
||||
|
||||
# optional |
||||
fake = { fake directives }; |
||||
fix = { fix directives }; |
||||
keep = { keep directives }; |
||||
# file to inject before first code-line of script |
||||
prologue = file; |
||||
# file to inject after last code-line of script |
||||
epilogue = file; |
||||
# extra command-line flags passed to resholve; generally this API |
||||
# should align with what resholve supports, but flags may help if |
||||
# you need to override the version of resholve. |
||||
flags = [ ]; |
||||
}; |
||||
}; |
||||
``` |
||||
|
||||
The main way you'll adjust how resholve handles your scripts are the |
||||
fake, fix, and keep directives. The manpage covers their purpose and |
||||
how to format them on the command-line, so I'll focus on how you'll |
||||
need to translate them into Nix types. |
||||
|
||||
```nix |
||||
# --fake 'f:setUp;tearDown builtin:setopt source:/etc/bashrc' |
||||
fake = { |
||||
function = [ "setUp" "tearDown" ]; |
||||
builtin = [ "setopt" ]; |
||||
source = [ "/etc/bashrc" ]; |
||||
}; |
||||
|
||||
# --fix 'aliases xargs:ls $GIT:gix' |
||||
fix = { |
||||
# all single-word directives use `true` as value |
||||
aliases = true; |
||||
xargs = [ "ls" ]; |
||||
"$GIT" = [ "gix" ]; |
||||
}; |
||||
|
||||
# --keep 'which:git;ls .:$HOME $LS:exa /etc/bashrc ~/.bashrc' |
||||
keep = { |
||||
which = [ "git" "ls" ]; |
||||
"." = [ "$HOME" ]; |
||||
"$LS" = [ "exa" ]; |
||||
"/etc/bashrc" = true; |
||||
"~/.bashrc" = true; |
||||
}; |
||||
``` |
@ -0,0 +1,9 @@ |
||||
{ callPackage |
||||
, doCheck ? true |
||||
}: |
||||
|
||||
rec { |
||||
resholve = callPackage ./resholve.nix { inherit doCheck; }; |
||||
resholvePackage = |
||||
callPackage ./resholve-package.nix { inherit resholve; }; |
||||
} |
@ -0,0 +1,120 @@ |
||||
{ stdenv |
||||
, python27Packages |
||||
, fetchFromGitHub |
||||
, makeWrapper |
||||
, # re2c deps |
||||
autoreconfHook |
||||
, # py-yajl deps |
||||
git |
||||
, # oil deps |
||||
readline |
||||
, cmark |
||||
, file |
||||
, glibcLocales |
||||
, oilPatches ? [ ] |
||||
}: |
||||
|
||||
/* |
||||
Notes on specific dependencies: |
||||
- if/when python2.7 is removed from nixpkgs, this may need to figure |
||||
out how to build oil's vendored python2 |
||||
- I'm not sure if glibcLocales is worth the addition here. It's to fix |
||||
a libc test oil runs. My oil fork just disabled the libc tests, but |
||||
I haven't quite decided if that's the right long-term call, so I |
||||
didn't add a patch for it here yet. |
||||
*/ |
||||
|
||||
rec { |
||||
# had to add this as well; 1.3 causes a break here; sticking |
||||
# to oil's official 1.0.3 dep for now. |
||||
re2c = stdenv.mkDerivation rec { |
||||
pname = "re2c"; |
||||
version = "1.0.3"; |
||||
sourceRoot = "${src.name}/re2c"; |
||||
src = fetchFromGitHub { |
||||
owner = "skvadrik"; |
||||
repo = "re2c"; |
||||
rev = version; |
||||
sha256 = "0grx7nl9fwcn880v5ssjljhcb9c5p2a6xpwil7zxpmv0rwnr3yqi"; |
||||
}; |
||||
nativeBuildInputs = [ autoreconfHook ]; |
||||
preCheck = '' |
||||
patchShebangs run_tests.sh |
||||
''; |
||||
}; |
||||
|
||||
py-yajl = python27Packages.buildPythonPackage rec { |
||||
pname = "oil-pyyajl-unstable"; |
||||
version = "2019-12-05"; |
||||
src = fetchFromGitHub { |
||||
owner = "oilshell"; |
||||
repo = "py-yajl"; |
||||
rev = "eb561e9aea6e88095d66abcc3990f2ee1f5339df"; |
||||
sha256 = "17hcgb7r7cy8r1pwbdh8di0nvykdswlqj73c85k6z8m0filj3hbh"; |
||||
fetchSubmodules = true; |
||||
}; |
||||
# just for submodule IIRC |
||||
nativeBuildInputs = [ git ]; |
||||
}; |
||||
|
||||
# resholve's primary dependency is this developer build of the oil shell. |
||||
oildev = python27Packages.buildPythonPackage rec { |
||||
pname = "oildev-unstable"; |
||||
version = "2020-03-31"; |
||||
|
||||
src = fetchFromGitHub { |
||||
owner = "oilshell"; |
||||
repo = "oil"; |
||||
rev = "ea80cdad7ae1152a25bd2a30b87fe3c2ad32394a"; |
||||
sha256 = "0pxn0f8qbdman4gppx93zwml7s5byqfw560n079v68qjgzh2brq2"; |
||||
|
||||
/* |
||||
It's not critical to drop most of these; the primary target is |
||||
the vendored fork of Python-2.7.13, which is ~ 55M and over 3200 |
||||
files, dozens of which get interpreter script patches in fixup. |
||||
*/ |
||||
extraPostFetch = '' |
||||
rm -rf Python-2.7.13 benchmarks metrics py-yajl rfc gold web testdata services demo devtools cpp |
||||
''; |
||||
}; |
||||
|
||||
# TODO: not sure why I'm having to set this for nix-build... |
||||
# can anyone tell if I'm doing something wrong? |
||||
SOURCE_DATE_EPOCH = 315532800; |
||||
|
||||
# These aren't, strictly speaking, nix/nixpkgs specific, but I've |
||||
# had hell upstreaming them. Pulling from resholve source and |
||||
# passing in from resholve.nix |
||||
patches = oilPatches; |
||||
|
||||
buildInputs = [ readline cmark py-yajl ]; |
||||
|
||||
nativeBuildInputs = [ re2c file makeWrapper ]; |
||||
|
||||
propagatedBuildInputs = with python27Packages; [ six typing ]; |
||||
|
||||
doCheck = true; |
||||
|
||||
preBuild = '' |
||||
build/dev.sh all |
||||
''; |
||||
|
||||
postPatch = '' |
||||
patchShebangs asdl build core doctools frontend native oil_lang |
||||
''; |
||||
|
||||
_NIX_SHELL_LIBCMARK = "${cmark}/lib/libcmark${stdenv.hostPlatform.extensions.sharedLibrary}"; |
||||
|
||||
# See earlier note on glibcLocales |
||||
LOCALE_ARCHIVE = stdenv.lib.optionalString (stdenv.buildPlatform.libc == "glibc") "${glibcLocales}/lib/locale/locale-archive"; |
||||
|
||||
meta = { |
||||
description = "A new unix shell"; |
||||
homepage = "https://www.oilshell.org/"; |
||||
license = with stdenv.lib.licenses; [ |
||||
psfl # Includes a portion of the python interpreter and standard library |
||||
asl20 # Licence for Oil itself |
||||
]; |
||||
}; |
||||
}; |
||||
} |
@ -0,0 +1,97 @@ |
||||
{ stdenv, lib, resholve }: |
||||
|
||||
{ pname |
||||
, src |
||||
, version |
||||
, passthru ? { } |
||||
, solutions |
||||
, ... |
||||
}@attrs: |
||||
let |
||||
inherit stdenv; |
||||
/* These functions break up the work of partially validating the |
||||
* 'solutions' attrset and massaging it into env/cli args. |
||||
* |
||||
* Note: some of the left-most args do not *have* to be passed as |
||||
* deep as they are, but I've done so to provide more error context |
||||
*/ |
||||
|
||||
# for brevity / line length |
||||
spaces = l: builtins.concatStringsSep " " l; |
||||
semicolons = l: builtins.concatStringsSep ";" l; |
||||
|
||||
/* Throw a fit with dotted attr path context */ |
||||
nope = path: msg: |
||||
throw "${builtins.concatStringsSep "." path}: ${msg}"; |
||||
|
||||
/* Special-case directive value representations by type */ |
||||
makeDirective = solution: env: name: val: |
||||
if builtins.isInt val then builtins.toString val |
||||
else if builtins.isString val then name |
||||
else if true == val then name |
||||
else if false == val then "" # omit! |
||||
else if null == val then "" # omit! |
||||
else if builtins.isList val then "${name}:${semicolons val}" |
||||
else nope [ solution env name ] "unexpected type: ${builtins.typeOf val}"; |
||||
|
||||
/* Build fake/fix/keep directives from Nix types */ |
||||
makeDirectives = solution: env: val: |
||||
lib.mapAttrsToList (makeDirective solution env) val; |
||||
|
||||
/* Special-case value representation by type/name */ |
||||
makeEnvVal = solution: env: val: |
||||
if env == "inputs" then lib.makeBinPath val |
||||
else if builtins.isString val then val |
||||
else if builtins.isList val then spaces val |
||||
else if builtins.isAttrs val then spaces (makeDirectives solution env val) |
||||
else nope [ solution env ] "unexpected type: ${builtins.typeOf val}"; |
||||
|
||||
/* Shell-format each env value */ |
||||
shellEnv = solution: env: value: |
||||
lib.escapeShellArg (makeEnvVal solution env value); |
||||
|
||||
/* Build a single ENV=val pair */ |
||||
makeEnv = solution: env: value: |
||||
"RESHOLVE_${lib.toUpper env}=${shellEnv solution env value}"; |
||||
|
||||
/* Discard attrs claimed by makeArgs */ |
||||
removeCliArgs = value: |
||||
removeAttrs value [ "scripts" "flags" ]; |
||||
|
||||
/* Verify required arguments are present */ |
||||
validateSolution = { scripts, inputs, interpreter, ... }: true; |
||||
|
||||
/* Pull out specific solution keys to build ENV=val pairs */ |
||||
makeEnvs = solution: value: |
||||
spaces (lib.mapAttrsToList (makeEnv solution) (removeCliArgs value)); |
||||
|
||||
/* Pull out specific solution keys to build CLI argstring */ |
||||
makeArgs = { flags ? [ ], scripts, ... }: |
||||
spaces (flags ++ scripts); |
||||
|
||||
/* Build a single resholve invocation */ |
||||
makeInvocation = solution: value: |
||||
if validateSolution value then |
||||
"${makeEnvs solution value} resholve --overwrite ${makeArgs value}" |
||||
else throw "invalid solution"; # shouldn't trigger for now |
||||
|
||||
/* Build resholve invocation for each solution. */ |
||||
makeCommands = solutions: |
||||
lib.mapAttrsToList makeInvocation solutions; |
||||
|
||||
self = (stdenv.mkDerivation ((removeAttrs attrs [ "solutions" ]) |
||||
// { |
||||
inherit pname version src; |
||||
buildInputs = [ resholve ]; |
||||
|
||||
# enable below for verbose debug info if needed |
||||
# supports default python.logging levels |
||||
# LOGLEVEL="INFO"; |
||||
preFixup = '' |
||||
pushd "$out" |
||||
${builtins.concatStringsSep "\n" (makeCommands solutions)} |
||||
popd |
||||
''; |
||||
})); |
||||
in |
||||
lib.extendDerivation true passthru self |
@ -0,0 +1,74 @@ |
||||
{ stdenv |
||||
, callPackage |
||||
, python27Packages |
||||
, installShellFiles |
||||
, fetchFromGitHub |
||||
, file |
||||
, findutils |
||||
, gettext |
||||
, bats |
||||
, bash |
||||
, doCheck ? true |
||||
}: |
||||
let |
||||
version = "0.4.0"; |
||||
rSrc = fetchFromGitHub { |
||||
owner = "abathur"; |
||||
repo = "resholve"; |
||||
rev = "v${version}"; |
||||
hash = "sha256-wfxcX3wMZqoi5bWjXYRa21UDDJmTDfE+21p4mL2IJog="; |
||||
}; |
||||
deps = callPackage ./deps.nix { |
||||
/* |
||||
resholve needs to patch Oil, but trying to avoid adding |
||||
them all *to* nixpkgs, since they aren't specific to |
||||
nix/nixpkgs. |
||||
*/ |
||||
oilPatches = [ |
||||
"${rSrc}/0001-add_setup_py.patch" |
||||
"${rSrc}/0002-add_MANIFEST_in.patch" |
||||
"${rSrc}/0003-fix_codegen_shebang.patch" |
||||
"${rSrc}/0004-disable-internal-py-yajl-for-nix-built.patch" |
||||
]; |
||||
}; |
||||
in |
||||
python27Packages.buildPythonApplication { |
||||
pname = "resholve"; |
||||
inherit version; |
||||
src = rSrc; |
||||
format = "other"; |
||||
|
||||
nativeBuildInputs = [ installShellFiles ]; |
||||
|
||||
propagatedBuildInputs = [ deps.oildev python27Packages.ConfigArgParse ]; |
||||
|
||||
patchPhase = '' |
||||
for file in resholve; do |
||||
substituteInPlace $file --subst-var-by version ${version} |
||||
done |
||||
''; |
||||
|
||||
installPhase = '' |
||||
install -Dm755 resholve $out/bin/resholve |
||||
installManPage resholve.1 |
||||
''; |
||||
|
||||
inherit doCheck; |
||||
checkInputs = [ bats ]; |
||||
RESHOLVE_PATH = "${stdenv.lib.makeBinPath [ file findutils gettext ]}"; |
||||
|
||||
checkPhase = '' |
||||
# explicit interpreter for test suite |
||||
export INTERP="${bash}/bin/bash" PATH="$out/bin:$PATH" |
||||
patchShebangs . |
||||
./test.sh |
||||
''; |
||||
|
||||
meta = with stdenv.lib; { |
||||
description = "Resolve external shell-script dependencies"; |
||||
homepage = "https://github.com/abathur/resholve"; |
||||
license = with licenses; [ mit ]; |
||||
maintainers = with maintainers; [ abathur ]; |
||||
platforms = platforms.all; |
||||
}; |
||||
} |
Loading…
Reference in new issue