ipfs-migrator: 1.7.1 -> 2.0.2

https://github.com/ipfs/fs-repo-migrations/releases/tag/v2.0.2

This is pretty much a complete rewrite of the ipfs-migrator package.
In version 2.0.0 a major change was made to the way the migrator works. Before, there was one binary that contained every migration. Now every migration has its own binary. If fs-repo-migrations can't find a required binary in the PATH, it will download it off the internet. To prevent that, build every migration individually, symlink them all into one package and then wrap fs-repo-migrations so it finds the package with all the migrations.
The change to the IPFS NixOS module and the IPFS package is needed because without explicitly specifying a repo version to migrate to, fs-repo-migrations will query the internet to find the latest version. This fails in the sandbox, for example when testing the ipfs passthru tests.
While it may seem like the repoVersion and IPFS version are in sync and the code could be simplified, this is not the case. See https://github.com/ipfs/fs-repo-migrations#when-should-i-migrate for a table with the IPFS versions and corresponding repo versions.
Go 1.17 breaks the migrations, so use Go 1.16 instead. This is also the Go version used in their CI, see 3dc218e300/.github/workflows/test.yml (L4). See https://github.com/ipfs/fs-repo-migrations/pull/140#issuecomment-982715907 for a previous mention of this issue. The issue manifests itself when doing anything with a migration, for example `fs-repo-11-to-12 --help`:
```
panic: qtls.ClientHelloInfo doesn't match

goroutine 1 [running]:
github.com/marten-seemann/qtls-go1-15.init.0()
	github.com/marten-seemann/qtls-go1-15@v0.1.1/unsafe.go:20 +0x132
```
Also add myself as a maintainer for this package.
This fixes the test failure discovered in https://github.com/NixOS/nixpkgs/pull/160914.
See https://github.com/ipfs/fs-repo-migrations/issues/148 to read some of my struggles with updating this package.
main
Luflosi 2 years ago committed by Jonathan Ringer
parent 1e7c64aae1
commit 0eb1d070c8
  1. 2
      nixos/modules/services/network-filesystems/ipfs.nix
  2. 63
      pkgs/applications/networking/ipfs-migrator/all-migrations.nix
  3. 36
      pkgs/applications/networking/ipfs-migrator/default.nix
  4. 35
      pkgs/applications/networking/ipfs-migrator/unwrapped.nix
  5. 4
      pkgs/applications/networking/ipfs/default.nix
  6. 7
      pkgs/top-level/all-packages.nix

@ -259,7 +259,7 @@ in
ipfs --offline config Mounts.IPFS ${cfg.ipfsMountDir}
ipfs --offline config Mounts.IPNS ${cfg.ipnsMountDir}
'' + optionalString cfg.autoMigrate ''
${pkgs.ipfs-migrator}/bin/fs-repo-migrations -y
${pkgs.ipfs-migrator}/bin/fs-repo-migrations -to '${cfg.package.repoVersion}' -y
'' + ''
ipfs --offline config show \
| ${pkgs.jq}/bin/jq '. * $extraConfig' --argjson extraConfig ${

@ -0,0 +1,63 @@
{ lib
, stdenv
, symlinkJoin
, buildGoModule
, ipfs-migrator-unwrapped
}:
# This package contains all the individual migrations in the bin directory.
# This is used by fs-repo-migrations and could also be used by IPFS itself
# when starting it like this: ipfs daemon --migrate
let
fs-repo-common = pname: version: buildGoModule {
inherit pname version;
inherit (ipfs-migrator-unwrapped) src;
sourceRoot = "source/${pname}";
vendorSha256 = null;
doCheck = false;
meta = ipfs-migrator-unwrapped.meta // {
mainProgram = pname;
description = "Individual migration for the filesystem repository of ipfs clients";
};
};
# Concatenation of the latest repo version and the version of that migration
version = "12.1.0.2";
fs-repo-11-to-12 = fs-repo-common "fs-repo-11-to-12" "1.0.2";
fs-repo-10-to-11 = fs-repo-common "fs-repo-10-to-11" "1.0.1";
fs-repo-9-to-10 = fs-repo-common "fs-repo-9-to-10" "1.0.1";
fs-repo-8-to-9 = fs-repo-common "fs-repo-8-to-9" "1.0.1";
fs-repo-7-to-8 = fs-repo-common "fs-repo-7-to-8" "1.0.1";
fs-repo-6-to-7 = fs-repo-common "fs-repo-6-to-7" "1.0.1";
fs-repo-5-to-6 = fs-repo-common "fs-repo-5-to-6" "1.0.1";
fs-repo-4-to-5 = fs-repo-common "fs-repo-4-to-5" "1.0.1";
fs-repo-3-to-4 = fs-repo-common "fs-repo-3-to-4" "1.0.1";
fs-repo-2-to-3 = fs-repo-common "fs-repo-2-to-3" "1.0.1";
fs-repo-1-to-2 = fs-repo-common "fs-repo-1-to-2" "1.0.1";
fs-repo-0-to-1 = fs-repo-common "fs-repo-0-to-1" "1.0.1";
all-migrations = [
fs-repo-11-to-12
fs-repo-10-to-11
fs-repo-9-to-10
fs-repo-8-to-9
fs-repo-7-to-8
] ++ lib.optional (!stdenv.isDarwin) # I didn't manage to fix this on macOS:
fs-repo-6-to-7 # gx/ipfs/QmSGRM5Udmy1jsFBr1Cawez7Lt7LZ3ZKA23GGVEsiEW6F3/eventfd/eventfd.go:27:32: undefined: syscall.SYS_EVENTFD2
++ [
fs-repo-5-to-6
fs-repo-4-to-5
fs-repo-3-to-4
fs-repo-2-to-3
fs-repo-1-to-2
fs-repo-0-to-1
];
in
symlinkJoin {
name = "ipfs-migrator-all-fs-repo-migrations-${version}";
paths = all-migrations;
}

@ -1,27 +1,23 @@
{ lib, buildGoModule, fetchFromGitHub }:
{ lib
, buildEnv
, makeWrapper
, ipfs-migrator-unwrapped
, ipfs-migrator-all-fs-repo-migrations
}:
buildGoModule rec {
pname = "ipfs-migrator";
version = "1.7.1";
buildEnv {
name = "ipfs-migrator-${ipfs-migrator-unwrapped.version}";
src = fetchFromGitHub {
owner = "ipfs";
repo = "fs-repo-migrations";
rev = "v${version}";
sha256 = "sha256-MxEKmoveIpuxBkGGGJHp9T11i3Py8a1fLpF0fWk0ftg=";
};
nativeBuildInputs = [ makeWrapper ];
vendorSha256 = null;
paths = [ ipfs-migrator-unwrapped ];
doCheck = false;
pathsToLink = [ "/bin" ];
subPackages = [ "." ];
postBuild = ''
wrapProgram "$out/bin/fs-repo-migrations" \
--prefix PATH ':' '${lib.makeBinPath [ ipfs-migrator-all-fs-repo-migrations ]}'
'';
meta = with lib; {
description = "Migrations for the filesystem repository of ipfs clients";
homepage = "https://ipfs.io/";
license = licenses.mit;
platforms = platforms.unix;
maintainers = with maintainers; [ elitak ];
};
inherit (ipfs-migrator-unwrapped) meta;
}

@ -0,0 +1,35 @@
{ lib
, buildGoModule
, fetchFromGitHub
}:
buildGoModule rec {
pname = "ipfs-migrator";
version = "2.0.2";
src = fetchFromGitHub {
owner = "ipfs";
repo = "fs-repo-migrations";
# Use the latest git tag here, since v2.0.2 does not
# contain the latest migration fs-repo-11-to-12/v1.0.2
# The fs-repo-migrations code itself is the same between
# the two versions but the migration code, which is built
# into separate binaries, is not.
rev = "fs-repo-11-to-12/v1.0.2";
sha256 = "sha256-CG4utwH+/+Igw+SP3imhl39wijlB53UGtkJG5Mwh+Ik=";
};
sourceRoot = "source/fs-repo-migrations";
vendorSha256 = "sha256-/DqkBBtR/nU8gk3TFqNKY5zQU6BFMc3N8Ti+38mi/jk=";
doCheck = false;
meta = with lib; {
description = "Migrations for the filesystem repository of ipfs clients";
homepage = "https://github.com/ipfs/fs-repo-migrations";
license = licenses.mit;
maintainers = with maintainers; [ Luflosi elitak ];
mainProgram = "fs-repo-migrations";
};
}

@ -2,9 +2,11 @@
buildGoModule rec {
pname = "ipfs";
version = "0.11.0";
version = "0.11.0"; # When updating, also check if the repo version changed and adjust repoVersion below
rev = "v${version}";
repoVersion = "11"; # Also update ipfs-migrator when changing the repo version
# go-ipfs makes changes to it's source tarball that don't match the git source.
src = fetchurl {
url = "https://github.com/ipfs/go-ipfs/releases/download/${rev}/go-ipfs-source.tar.gz";

@ -6721,12 +6721,15 @@ with pkgs;
ipfs = callPackage ../applications/networking/ipfs {
buildGoModule = buildGo116Module;
};
ipfs-migrator = callPackage ../applications/networking/ipfs-migrator {
ipfs-cluster = callPackage ../applications/networking/ipfs-cluster {
buildGoModule = buildGo116Module;
};
ipfs-cluster = callPackage ../applications/networking/ipfs-cluster {
ipfs-migrator-all-fs-repo-migrations = callPackage ../applications/networking/ipfs-migrator/all-migrations.nix {
buildGoModule = buildGo116Module;
};
ipfs-migrator-unwrapped = callPackage ../applications/networking/ipfs-migrator/unwrapped.nix { };
ipfs-migrator = callPackage ../applications/networking/ipfs-migrator { };
ipget = callPackage ../applications/networking/ipget { };

Loading…
Cancel
Save