nixos/rdma-core: add module for soft RoCE and test (#34607)
parent
58cf763f8b
commit
bf53dc68c2
@ -0,0 +1,63 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
cfg = config.networking.rxe; |
||||
|
||||
runRxeCmd = cmd: ifcs: |
||||
concatStrings ( map (x: "${pkgs.rdma-core}/bin/rxe_cfg -n ${cmd} ${x};") ifcs); |
||||
|
||||
startScript = pkgs.writeShellScriptBin "rxe-start" '' |
||||
${pkgs.rdma-core}/bin/rxe_cfg -n start |
||||
${runRxeCmd "add" cfg.interfaces} |
||||
${pkgs.rdma-core}/bin/rxe_cfg |
||||
''; |
||||
|
||||
stopScript = pkgs.writeShellScriptBin "rxe-stop" '' |
||||
${runRxeCmd "remove" cfg.interfaces } |
||||
${pkgs.rdma-core}/bin/rxe_cfg -n stop |
||||
''; |
||||
|
||||
in { |
||||
###### interface |
||||
|
||||
options = { |
||||
networking.rxe = { |
||||
enable = mkEnableOption "RDMA over converged ethernet"; |
||||
interfaces = mkOption { |
||||
type = types.listOf types.str; |
||||
default = [ ]; |
||||
example = [ "eth0" ]; |
||||
description = '' |
||||
Enable RDMA on the listed interfaces. The corresponding virtual |
||||
RDMA interfaces will be named rxe0 ... rxeN where the ordering |
||||
will be as they are named in the list. UDP port 4791 must be |
||||
open on the respective ethernet interfaces. |
||||
''; |
||||
}; |
||||
}; |
||||
}; |
||||
|
||||
###### implementation |
||||
|
||||
config = mkIf cfg.enable { |
||||
|
||||
systemd.services.rxe = { |
||||
path = with pkgs; [ kmod rdma-core ]; |
||||
description = "RoCE interfaces"; |
||||
|
||||
wantedBy = [ "multi-user.target" ]; |
||||
after = [ "systemd-modules-load.service" "network-online.target" ]; |
||||
wants = [ "network-pre.target" ]; |
||||
|
||||
serviceConfig = { |
||||
Type = "oneshot"; |
||||
RemainAfterExit = true; |
||||
ExecStart = "${startScript}/bin/rxe-start"; |
||||
ExecStop = "${stopScript}/bin/rxe-stop"; |
||||
}; |
||||
}; |
||||
}; |
||||
} |
||||
|
@ -0,0 +1,53 @@ |
||||
import ./make-test.nix ({ pkgs, ... } : |
||||
|
||||
let |
||||
node = { config, pkgs, lib, ... } : { |
||||
networking = { |
||||
firewall = { |
||||
allowedUDPPorts = [ 4791 ]; # open RoCE port |
||||
allowedTCPPorts = [ 4800 ]; # port for test utils |
||||
}; |
||||
rxe = { |
||||
enable = true; |
||||
interfaces = [ "eth1" ]; |
||||
}; |
||||
}; |
||||
|
||||
environment.systemPackages = with pkgs; [ rdma-core screen ]; |
||||
}; |
||||
|
||||
in { |
||||
name = "rxe"; |
||||
|
||||
nodes = { |
||||
server = node; |
||||
client = node; |
||||
}; |
||||
|
||||
testScript = '' |
||||
# Test if rxe interface comes up |
||||
$server->waitForUnit("default.target"); |
||||
$server->succeed("systemctl status rxe.service"); |
||||
$server->succeed("ibv_devices | grep rxe0"); |
||||
|
||||
$client->waitForUnit("default.target"); |
||||
|
||||
# ping pong test |
||||
$server->succeed("screen -dmS rc_pingpong ibv_rc_pingpong -p 4800 -g0"); |
||||
$client->succeed("sleep 2; ibv_rc_pingpong -p 4800 -g0 server"); |
||||
|
||||
$server->succeed("screen -dmS uc_pingpong ibv_uc_pingpong -p 4800 -g0"); |
||||
$client->succeed("sleep 2; ibv_uc_pingpong -p 4800 -g0 server"); |
||||
|
||||
$server->succeed("screen -dmS ud_pingpong ibv_ud_pingpong -p 4800 -s 1024 -g0"); |
||||
$client->succeed("sleep 2; ibv_ud_pingpong -p 4800 -s 1024 -g0 server"); |
||||
|
||||
$server->succeed("screen -dmS srq_pingpong ibv_srq_pingpong -p 4800 -g0"); |
||||
$client->succeed("sleep 2; ibv_srq_pingpong -p 4800 -g0 server"); |
||||
|
||||
$server->succeed("screen -dmS rping rping -s -a server -C 10"); |
||||
$client->succeed("sleep 2; rping -c -a server -C 10"); |
||||
''; |
||||
}) |
||||
|
||||
|
Loading…
Reference in new issue