tree-sitter: init at 0.15.7

pkgs/development/tools/parsing/tree-sitter/default.nix

@@ -0,0 +1,106 @@
+{ lib, stdenv
+, fetchgit, fetchFromGitHub, fetchurl
+, writeShellScript, runCommand
+, rustPlatform, jq, nix-prefetch-git, xe, curl
+}:
+
+# TODO: move to carnix or https://github.com/kolloch/crate2nix
+let
+  # to update:
+  # 1) change all these hashes
+  # 2) nix-build -A tree-sitter.updater.update-all-grammars
+  # 3) run the script that is output by that (it updates ./grammars)
+  version = "0.15.7";
+  sha256 = "0q6w8wl4a4s49xlgbv531pandzrj3n12hc1cwfshzcgikx303dg0";
+  sha256Js = "11ig4cc2m85siyhafh4hq9sjb5if4gfwsf9k87izkxpiyflda0wp";
+  sha256Wasm = "1zm4bvjri8ivhah3sy22mx6jbvibgbn2hk67d148j3nyka3y4gc0";
+  cargoSha256 = "0ls9cb2p6cgqvnrmx72n79ga7687n8mzhh7n8n1pzv11r6cah9ki";
+
+
+  src = fetchFromGitHub {
+    owner = "tree-sitter";
+    repo = "tree-sitter";
+    rev = version;
+    inherit sha256;
+    fetchSubmodules = true;
+  };
+
+  fetchDist = {file, sha256}: fetchurl {
+    url = "https://github.com/tree-sitter/tree-sitter/releases/download/${version}/${file}";
+    inherit sha256;
+  };
+
+  # TODO: not distributed anymore; needed for the web-ui module,
+  # see also the disable-web-ui patch.
+  # TODO: build those instead of downloading prebuilt
+  # js = fetchDist {
+  #   file = "tree-sitter.js";
+  #   sha256 = sha256Js;
+  # };
+  # wasm = fetchDist {
+  #   file = "tree-sitter.wasm";
+  #   sha256 = sha256Wasm;
+  # };
+
+  update-all-grammars = import ./update.nix {
+    inherit writeShellScript nix-prefetch-git curl jq xe src;
+  };
+
+  grammars =
+    let fetch =
+      (v: fetchgit {inherit (v) url rev sha256 fetchSubmodules; });
+    in runCommand "grammars" {} (''
+       mkdir $out
+     '' + (lib.concatStrings (lib.mapAttrsToList
+            (name: grammar: "ln -s ${fetch grammar} $out/${name}\n")
+            (import ./grammars))));
+
+
+in rustPlatform.buildRustPackage {
+  pname = "tree-sitter";
+  inherit version;
+  inherit src;
+
+  patches = [
+    # the web ui requires tree-sitter compiled to js and wasm
+    ./disable-web-ui.patch
+  ];
+
+  postPatch = ''
+    # needed for the tests
+    rm -rf test/fixtures/grammars
+    ln -s ${grammars} test/fixtures/grammars
+  '';
+
+  passthru = {
+    updater = {
+      inherit update-all-grammars;
+    };
+    inherit grammars;
+  };
+
+  inherit cargoSha256;
+
+  meta = {
+    homepage = "https://github.com/tree-sitter/tree-sitter";
+    description = "A parser generator tool and an incremental parsing library";
+    longDescription = ''
+      Tree-sitter is a parser generator tool and an incremental parsing library.
+      It can build a concrete syntax tree for a source file and efficiently update the syntax tree as the source file is edited.
+
+      Tree-sitter aims to be:
+
+      * General enough to parse any programming language
+      * Fast enough to parse on every keystroke in a text editor
+      * Robust enough to provide useful results even in the presence of syntax errors
+      * Dependency-free so that the runtime library (which is written in pure C) can be embedded in any application
+    '';
+    platforms = lib.platforms.all;
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ Profpatsch ];
+    # Darwin needs some more work with default libraries
+    # Aarch has test failures with how tree-sitter compiles the generated C files
+    broken = stdenv.isDarwin || stdenv.isAarch64;
+  };
+
+}

pkgs/development/tools/parsing/tree-sitter/disable-web-ui.patch

@@ -0,0 +1,37 @@
+diff --git a/cli/src/lib.rs b/cli/src/lib.rs
+index 33a9904f..633032d7 100644
+--- a/cli/src/lib.rs
++++ b/cli/src/lib.rs
+@@ -8,7 +8,7 @@ pub mod parse;
+ pub mod test;
+ pub mod util;
+ pub mod wasm;
+-pub mod web_ui;
++// pub mod web_ui;
+ 
+ #[cfg(test)]
+ mod tests;
+diff --git a/cli/src/main.rs b/cli/src/main.rs
+index 23e7fc1a..9d784c8a 100644
+--- a/cli/src/main.rs
++++ b/cli/src/main.rs
+@@ -4,7 +4,7 @@ use std::{env, fs, u64};
+ use std::path::Path;
+ use std::process::exit;
+ use tree_sitter_cli::{
+-    config, error, generate, highlight, loader, logger, parse, test, wasm, web_ui,
++    config, error, generate, highlight, loader, logger, parse, test, wasm,
+ };
+ 
+ const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION");
+@@ -250,7 +250,9 @@ fn run() -> error::Result<()> {
+         let grammar_path = current_dir.join(matches.value_of("path").unwrap_or(""));
+         wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?;
+     } else if matches.subcommand_matches("web-ui").is_some() {
+-        web_ui::serve(&current_dir);
++        print!("ERROR: web-ui is not available in the nixpkgs tree-sitter-cli at the moment.");
++        std::process::exit(1);
++        // web_ui::serve(&current_dir);
+     }
+ 
+     Ok(())

pkgs/development/tools/parsing/tree-sitter/grammars/default.nix

@@ -0,0 +1,15 @@
+{
+  bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
+  c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
+  cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
+  embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
+  go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
+  html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
+  javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
+  json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
+  python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
+  # wasn’t able to check out with fetchgit
+  # ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
+  rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
+  typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-bash.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-bash",
+  "rev": "7b8adcf484e27b6c1a707ce28123e55dd4b7a844",
+  "date": "2019-07-26T14:05:41-06:00",
+  "sha256": "047p51ab4fqm55xqss6z74iyj1hlndql97dv9fifckczx3d5xn5g",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-c",
+  "rev": "22decdc361767838dd36f1da4125b35b5b9a3c28",
+  "date": "2019-07-02T15:49:42-07:00",
+  "sha256": "03f9g49l4g2l4hlafr3xhvi8d3a491k5zz4bxpq7391l5wgjy3zi",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-cpp",
+  "rev": "f5afa0ee48ad1dc067ed6fe1aa2cfd2a3ea5d443",
+  "date": "2019-08-06T17:23:46-07:00",
+  "sha256": "1w9zjqj232fcagqfqd8qi4kmvr655s4ivllrm27973sda4xq557h",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-embedded-template.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-embedded-template",
+  "rev": "71955edec8cb762f63e94cf062fc96b52b9ae609",
+  "date": "2019-07-17T15:55:22-07:00",
+  "sha256": "1ar2n1z2h194lb3isbdkmvhn8w78j4a62nbh105w3jl1sxb4qpsa",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-go.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-go",
+  "rev": "475571bb5bdb9b229c6be3843d4c71ba747688fd",
+  "date": "2019-07-17T15:51:06-07:00",
+  "sha256": "1cg5qpifrvpnsi0iy26g156xib2qa55vlna41hw6c70kx8ibvl9z",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-html.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-html",
+  "rev": "aeb2f456b8c6a60b8475d075889d476a165cde57",
+  "date": "2019-07-17T15:57:54-07:00",
+  "sha256": "0ba8zi65kja6p7f5h7pa7kxqa3mj29ysjrvl84am24vy5ik4zz3z",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-javascript",
+  "rev": "a730b5c210904e2e3c1f601125a059fde1b35850",
+  "date": "2019-08-08T14:13:17-07:00",
+  "sha256": "1cr0vikbzrklksjj07fh34a5cabkgbpkbxwiw2alnana3zzzdhnq",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-json.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-json",
+  "rev": "337f55be9b9b1ccb0baa7763bfe014a94acea7ea",
+  "date": "2019-05-23T11:02:26-04:00",
+  "sha256": "0amh4qrjj3fli9c0z6p61z9d7496sqq54i1gh2vrghgnbbyaa6mz",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-python",
+  "rev": "4c22de0944cd42a5c86ade7ef7097033604796f8",
+  "date": "2019-07-30T15:35:03-04:00",
+  "sha256": "1p12h7hj1ak15fyk4gw9wcmgzydd4z5mikhjp54mn1q4vfw175p3",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-ruby",
+  "rev": "db91c934ff9d3d4ea67111a0f581532c49c3a6b3",
+  "date": "2019-07-26T15:51:54-06:00",
+  "sha256": "1ir1nqpz0c0hnsqzp90w2iw1gy3z3nqil2fm4n3zmid5di7c98dg",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-rust",
+  "rev": "3f956b18a6b0a576ed238cc69d5e3f413bd547b1",
+  "date": "2019-07-18T11:44:02-07:00",
+  "sha256": "0dwxg3pqirqm1lvl5x0q9djavfri9ffk5diygqzjnx53rwqhyzj8",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json

@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-typescript",
+  "rev": "ab9ab6cced868ee3e096f33fa21fd9d356c92e1a",
+  "date": "2019-08-08T14:27:32-07:00",
+  "sha256": "11r0vj1dhv0my2cr442mwvaav8ljygsns20w51mwg7328vlz90q3",
+  "fetchSubmodules": false
+}

pkgs/development/tools/parsing/tree-sitter/update.nix

@@ -0,0 +1,68 @@
+{ writeShellScript, nix-prefetch-git
+, curl, jq, xe
+, src }:
+
+let
+  # print all the grammar names mentioned in the fetch-fixtures script
+  getGrammarNames = writeShellScript "get-grammars.sh" ''
+    set -euo pipefail
+    sed -ne 's/^fetch_grammar \(\S*\).*$/\1/p' \
+      ${src}/script/fetch-fixtures
+  '';
+
+  # TODO
+  urlEscape = x: x;
+  # TODO
+  urlEscapeSh = writeShellScript "escape-url" ''printf '%s' "$1"'';
+
+  # generic bash script to find the latest github release for a repo
+  latestGithubRelease = { owner }: writeShellScript "latest-github-release" ''
+    set -euo pipefail
+    repo="$1"
+    res=$(${curl}/bin/curl \
+      --silent \
+      "https://api.github.com/repos/${urlEscape owner}/$(${urlEscapeSh} "$repo")/releases/latest")
+    if [[ "$(printf "%s" "$res" | ${jq}bin/jq '.message')" =~ "rate limit" ]]; then
+      echo "rate limited" >&2
+    fi
+    release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name')
+    # github sometimes returns an empty list even tough there are releases
+    if [ "$release" = "null" ]; then
+      echo "uh-oh, latest for $repo is not there, using HEAD" >&2
+      release="HEAD"
+    fi
+    echo "$release"
+  '';
+
+  # update one tree-sitter grammar repo and print their nix-prefetch-git output
+  updateGrammar = { owner }: writeShellScript "update-grammar.sh" ''
+    set -euo pipefail
+    repo="$1"
+    latest="$(${latestGithubRelease { inherit owner; }} "$repo")"
+    echo "Fetching latest release ($latest) of $repo …" >&2
+    ${nix-prefetch-git}/bin/nix-prefetch-git \
+      --quiet \
+      --no-deepClone \
+      --url "https://github.com/${urlEscape owner}/$(${urlEscapeSh} "$repo")" \
+      --rev "$latest"
+    '';
+
+  update-all-grammars = writeShellScript "update-all-grammars.sh" ''
+    set -euo pipefail
+    grammarNames=$(${getGrammarNames})
+    outputDir="${toString ./.}/grammars"
+    mkdir -p "$outputDir"
+    updateCommand=$(printf \
+      '${updateGrammar { owner = "tree-sitter"; }} "$1" > "%s/$1.json"' \
+      "$outputDir")
+    printf '%s' "$grammarNames" \
+      | ${xe}/bin/xe printf "tree-sitter-%s\n" {} \
+      | ${xe}/bin/xe -j2 -s "$updateCommand"
+    ( echo "{"
+      printf '%s' "$grammarNames" \
+        | ${xe}/bin/xe -s 'printf "  %s = (builtins.fromJSON (builtins.readFile ./tree-sitter-%s.json));\n" "$1" "$1"'
+      echo "}" ) \
+      > "$outputDir/default.nix"
+  '';
+
+in update-all-grammars

pkgs/top-level/all-packages.nix

@@ -10067,6 +10067,8 @@ in
 
   travis = callPackage ../development/tools/misc/travis { };
 
+  tree-sitter = callPackage ../development/tools/parsing/tree-sitter { };
+
   trellis = callPackage ../development/tools/trellis { };
 
   ttyd = callPackage ../servers/ttyd { };