The EBS and S3 (instance-store) AMIs are now created from the same image. HVM instance-store AMIs are also generated. Disk image generation has been factored out into a function (nixos/lib/make-disk-image.nix) that can be used to build other kinds of images.wip/yesman
Eelco Dolstra
9 years ago
parent
efed00b55e
commit
aeb31b97ad
@ -0,0 +1,105 @@ |
||||
{ pkgs |
||||
, lib |
||||
|
||||
, # The NixOS configuration to be installed onto the disk image. |
||||
config |
||||
|
||||
, # The size of the disk, in megabytes. |
||||
diskSize |
||||
|
||||
, # Whether the disk should be partitioned (with a single partition |
||||
# containing the root filesystem) or contain the root filesystem |
||||
# directly. |
||||
partitioned ? true |
||||
|
||||
, # The root file system type. |
||||
fsType ? "ext4" |
||||
|
||||
, # The initial NixOS configuration file to be copied to |
||||
# /etc/nixos/configuration.nix. |
||||
configFile ? null |
||||
|
||||
}: |
||||
|
||||
with lib; |
||||
|
||||
pkgs.vmTools.runInLinuxVM ( |
||||
pkgs.runCommand "nixos-disk-image" |
||||
{ preVM = |
||||
'' |
||||
mkdir $out |
||||
diskImage=$out/nixos.img |
||||
${pkgs.vmTools.qemu}/bin/qemu-img create -f raw $diskImage "${toString diskSize}M" |
||||
mv closure xchg/ |
||||
''; |
||||
buildInputs = [ pkgs.utillinux pkgs.perl pkgs.e2fsprogs pkgs.parted ]; |
||||
exportReferencesGraph = |
||||
[ "closure" config.system.build.toplevel ]; |
||||
} |
||||
'' |
||||
${if partitioned then '' |
||||
# Create a single / partition. |
||||
parted /dev/vda mklabel msdos |
||||
parted /dev/vda -- mkpart primary ext2 1M -1s |
||||
. /sys/class/block/vda1/uevent |
||||
mknod /dev/vda1 b $MAJOR $MINOR |
||||
rootDisk=/dev/vda1 |
||||
'' else '' |
||||
rootDisk=/dev/vda |
||||
''} |
||||
|
||||
# Create an empty filesystem and mount it. |
||||
mkfs.${fsType} -L nixos $rootDisk |
||||
${optionalString (fsType == "ext4") '' |
||||
tune2fs -c 0 -i 0 $rootDisk |
||||
''} |
||||
mkdir /mnt |
||||
mount $rootDisk /mnt |
||||
|
||||
# The initrd expects these directories to exist. |
||||
mkdir /mnt/dev /mnt/proc /mnt/sys |
||||
|
||||
mount -o bind /proc /mnt/proc |
||||
mount -o bind /dev /mnt/dev |
||||
mount -o bind /sys /mnt/sys |
||||
|
||||
# Copy all paths in the closure to the filesystem. |
||||
storePaths=$(perl ${pkgs.pathsFromGraph} /tmp/xchg/closure) |
||||
|
||||
mkdir -p /mnt/nix/store |
||||
echo "copying everything (will take a while)..." |
||||
cp -prd $storePaths /mnt/nix/store/ |
||||
|
||||
# Register the paths in the Nix database. |
||||
printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \ |
||||
chroot /mnt ${config.nix.package}/bin/nix-store --load-db --option build-users-group "" |
||||
|
||||
# Create the system profile to allow nixos-rebuild to work. |
||||
chroot /mnt ${config.nix.package}/bin/nix-env --option build-users-group "" \ |
||||
-p /nix/var/nix/profiles/system --set ${config.system.build.toplevel} |
||||
|
||||
# `nixos-rebuild' requires an /etc/NIXOS. |
||||
mkdir -p /mnt/etc |
||||
touch /mnt/etc/NIXOS |
||||
|
||||
# `switch-to-configuration' requires a /bin/sh |
||||
mkdir -p /mnt/bin |
||||
ln -s ${config.system.build.binsh}/bin/sh /mnt/bin/sh |
||||
|
||||
# Install a configuration.nix. |
||||
mkdir -p /mnt/etc/nixos |
||||
${optionalString (configFile != null) '' |
||||
cp ${configFile} /mnt/etc/nixos/configuration.nix |
||||
''} |
||||
|
||||
# Generate the GRUB menu. |
||||
ln -s vda /dev/xvda |
||||
chroot /mnt ${config.system.build.toplevel}/bin/switch-to-configuration boot |
||||
|
||||
umount /mnt/proc /mnt/dev /mnt/sys |
||||
umount /mnt |
||||
|
||||
# Do an fsck to make sure resize2fs works. |
||||
fsck.${fsType} -f -y $rootDisk |
||||
'' |
||||
) |
||||
@ -1,5 +0,0 @@ |
||||
{ modulesPath, ...}: |
||||
{ |
||||
imports = [ "${modulesPath}/virtualisation/amazon-init.nix" ]; |
||||
services.journald.rateLimitBurst = 0; |
||||
} |
||||
@ -1,5 +0,0 @@ |
||||
{ config, pkgs, ...}: |
||||
{ |
||||
imports = [ ./amazon-base-config.nix ]; |
||||
ec2.hvm = true; |
||||
} |
||||
@ -1,34 +0,0 @@ |
||||
{ config, pkgs, lib, ...}: |
||||
let |
||||
cloudUtils = pkgs.fetchurl { |
||||
url = "https://launchpad.net/cloud-utils/trunk/0.27/+download/cloud-utils-0.27.tar.gz"; |
||||
sha256 = "16shlmg36lidp614km41y6qk3xccil02f5n3r4wf6d1zr5n4v8vd"; |
||||
}; |
||||
growpart = pkgs.stdenv.mkDerivation { |
||||
name = "growpart"; |
||||
src = cloudUtils; |
||||
buildPhase = '' |
||||
cp bin/growpart $out |
||||
sed -i 's|awk|gawk|' $out |
||||
sed -i 's|sed|gnused|' $out |
||||
''; |
||||
dontInstall = true; |
||||
dontPatchShebangs = true; |
||||
}; |
||||
in |
||||
{ |
||||
imports = [ ./amazon-base-config.nix ]; |
||||
ec2.hvm = true; |
||||
boot.loader.grub.device = lib.mkOverride 0 "/dev/xvdg"; |
||||
boot.kernelParams = [ "console=ttyS0" ]; |
||||
|
||||
boot.initrd.extraUtilsCommands = '' |
||||
copy_bin_and_libs ${pkgs.gawk}/bin/gawk |
||||
copy_bin_and_libs ${pkgs.gnused}/bin/sed |
||||
copy_bin_and_libs ${pkgs.utillinux}/sbin/sfdisk |
||||
cp -v ${growpart} $out/bin/growpart |
||||
''; |
||||
boot.initrd.postDeviceCommands = '' |
||||
[ -e /dev/xvda ] && [ -e /dev/xvda1 ] && TMPDIR=/run sh $(type -P growpart) /dev/xvda 1 |
||||
''; |
||||
} |
||||
@ -0,0 +1,27 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
{ |
||||
|
||||
imports = |
||||
[ ../../../modules/installer/cd-dvd/channel.nix |
||||
../../../modules/virtualisation/amazon-image.nix |
||||
]; |
||||
|
||||
system.build.amazonImage = import ../../../lib/make-disk-image.nix { |
||||
inherit pkgs lib config; |
||||
partitioned = config.ec2.hvm; |
||||
diskSize = 8192; |
||||
configFile = pkgs.writeText "configuration.nix" |
||||
'' |
||||
{ |
||||
imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ]; |
||||
${optionalString config.ec2.hvm '' |
||||
ec2.hvm = true; |
||||
''} |
||||
} |
||||
''; |
||||
}; |
||||
|
||||
} |
||||
@ -0,0 +1,217 @@ |
||||
#! /bin/sh -e |
||||
|
||||
set -o pipefail |
||||
#set -x |
||||
|
||||
stateDir=${TMPDIR:-/tmp}/ec2-image |
||||
echo "keeping state in $stateDir" |
||||
mkdir -p $stateDir |
||||
|
||||
version=$(nix-instantiate --eval --strict '<nixpkgs>' -A lib.nixpkgsVersion | sed s/'"'//g) |
||||
echo "NixOS version is $version" |
||||
|
||||
rm -f ec2-amis.nix |
||||
|
||||
|
||||
for type in hvm pv; do |
||||
link=$stateDir/$type |
||||
imageFile=$link/nixos.img |
||||
system=x86_64-linux |
||||
arch=x86_64 |
||||
|
||||
# Build the image. |
||||
if ! [ -L $link ]; then |
||||
if [ $type = pv ]; then hvmFlag=false; else hvmFlag=true; fi |
||||
|
||||
echo "building image type '$type'..." |
||||
nix-build -o $link \ |
||||
'<nixpkgs/nixos>' \ |
||||
-A config.system.build.amazonImage \ |
||||
--arg configuration "{ imports = [ <nixpkgs/nixos/maintainers/scripts/ec2/amazon-image.nix> ]; ec2.hvm = $hvmFlag; }" |
||||
fi |
||||
|
||||
for store in ebs s3; do |
||||
|
||||
bucket=nixos-amis |
||||
bucketDir="$version-$type-$store" |
||||
|
||||
prevAmi= |
||||
prevRegion= |
||||
|
||||
#for region in eu-west-1 eu-central-1 us-east-1 us-west-1 us-west-2 ap-southeast-1 ap-southeast-2 ap-northeast-1 sa-east-1; do |
||||
for region in eu-west-1 us-east-1; do |
||||
|
||||
name=nixos-$version-$arch-$type-$store |
||||
description="NixOS $system $version ($type-$store)" |
||||
|
||||
amiFile=$stateDir/$region.$type.$store.ami-id |
||||
|
||||
if ! [ -e $amiFile ]; then |
||||
|
||||
echo "doing $name in $region..." |
||||
|
||||
if [ -n "$prevAmi" ]; then |
||||
ami=$(ec2-copy-image \ |
||||
--region "$region" \ |
||||
--source-region "$prevRegion" --source-ami-id "$prevAmi" \ |
||||
--name "$name" --description "$description" | cut -f 2) |
||||
else |
||||
|
||||
if [ $store = s3 ]; then |
||||
|
||||
# Bundle the image. |
||||
imageDir=$stateDir/$type-bundled |
||||
|
||||
if ! [ -d $imageDir ]; then |
||||
rm -rf $imageDir.tmp |
||||
mkdir -p $imageDir.tmp |
||||
ec2-bundle-image \ |
||||
-d $imageDir.tmp \ |
||||
-i $imageFile --arch $arch \ |
||||
--user "$AWS_ACCOUNT" -c "$EC2_CERT" -k "$EC2_PRIVATE_KEY" |
||||
mv $imageDir.tmp $imageDir |
||||
fi |
||||
|
||||
# Upload the bundle to S3. |
||||
if ! [ -e $imageDir/uploaded ]; then |
||||
echo "uploading bundle to S3..." |
||||
ec2-upload-bundle \ |
||||
-m $imageDir/nixos.img.manifest.xml \ |
||||
-b "$bucket/$bucketDir" \ |
||||
-a "$EC2_ACCESS_KEY" -s "$EC2_SECRET_KEY" \ |
||||
--location EU |
||||
touch $imageDir/uploaded |
||||
fi |
||||
|
||||
extraFlags="$bucket/$bucketDir/nixos.img.manifest.xml" |
||||
|
||||
else |
||||
|
||||
# Convert the image to vhd format so we don't have |
||||
# to upload a huge raw image. |
||||
vhdFile=$stateDir/$type.vhd |
||||
if ! [ -e $vhdFile ]; then |
||||
qemu-img convert -O vpc $imageFile $vhdFile.tmp |
||||
mv $vhdFile.tmp $vhdFile |
||||
fi |
||||
|
||||
taskId=$(cat $stateDir/$region.$type.task-id 2> /dev/null || true) |
||||
volId=$(cat $stateDir/$region.$type.vol-id 2> /dev/null || true) |
||||
snapId=$(cat $stateDir/$region.$type.snap-id 2> /dev/null || true) |
||||
|
||||
# Import the VHD file. |
||||
if [ -z "$snapId" -a -z "$volId" -a -z "$taskId" ]; then |
||||
echo "importing $vhdFile..." |
||||
taskId=$(ec2-import-volume $vhdFile --no-upload -f vhd \ |
||||
-o "$EC2_ACCESS_KEY" -w "$EC2_SECRET_KEY" \ |
||||
--region "$region" -z "${region}a" \ |
||||
--bucket "$bucket" --prefix "$bucketDir/" \ |
||||
| tee /dev/stderr \ |
||||
| sed 's/.*\(import-vol-[0-9a-z]\+\).*/\1/ ; t ; d') |
||||
echo -n "$taskId" > $stateDir/$region.$type.task-id |
||||
fi |
||||
|
||||
if [ -z "$snapId" -a -z "$volId" ]; then |
||||
ec2-resume-import $vhdFile -t "$taskId" --region "$region" \ |
||||
-o "$EC2_ACCESS_KEY" -w "$EC2_SECRET_KEY" |
||||
fi |
||||
|
||||
# Wait for the volume creation to finish. |
||||
if [ -z "$snapId" -a -z "$volId" ]; then |
||||
echo "waiting for import to finish..." |
||||
while true; do |
||||
volId=$(ec2-describe-conversion-tasks "$taskId" --region "$region" | sed 's/.*VolumeId.*\(vol-[0-9a-f]\+\).*/\1/ ; t ; d') |
||||
if [ -n "$volId" ]; then break; fi |
||||
sleep 10 |
||||
done |
||||
|
||||
echo -n "$volId" > $stateDir/$region.$type.vol-id |
||||
fi |
||||
|
||||
# Delete the import task. |
||||
if [ -n "$volId" -a -n "$taskId" ]; then |
||||
echo "removing import task..." |
||||
ec2-delete-disk-image -t "$taskId" --region "$region" -o "$EC2_ACCESS_KEY" -w "$EC2_SECRET_KEY" || true |
||||
rm -f $stateDir/$region.$type.task-id |
||||
fi |
||||
|
||||
# Create a snapshot. |
||||
if [ -z "$snapId" ]; then |
||||
echo "creating snapshot..." |
||||
snapId=$(ec2-create-snapshot "$volId" --region "$region" | cut -f 2) |
||||
echo -n "$snapId" > $stateDir/$region.$type.snap-id |
||||
ec2-create-tags "$snapId" -t "Name=$description" --region "$region" |
||||
fi |
||||
|
||||
# Wait for the snapshot to finish. |
||||
echo "waiting for snapshot to finish..." |
||||
while true; do |
||||
status=$(ec2-describe-snapshots "$snapId" --region "$region" | head -n1 | cut -f 4) |
||||
if [ "$status" = completed ]; then break; fi |
||||
sleep 10 |
||||
done |
||||
|
||||
# Delete the volume. |
||||
if [ -n "$volId" ]; then |
||||
echo "deleting volume..." |
||||
ec2-delete-volume "$volId" --region "$region" || true |
||||
rm -f $stateDir/$region.$type.vol-id |
||||
fi |
||||
|
||||
extraFlags="-b /dev/sda1=$snapId:20:true:gp2" |
||||
|
||||
if [ $type = pv ]; then |
||||
extraFlags+=" --root-device-name=/dev/sda1" |
||||
fi |
||||
|
||||
extraFlags+=" -b /dev/sdb=ephemeral0 -b /dev/sdc=ephemeral1 -b /dev/sdd=ephemeral2 -b /dev/sde=ephemeral3" |
||||
fi |
||||
|
||||
# Register the AMI. |
||||
if [ $type = pv ]; then |
||||
kernel=$(ec2-describe-images -o amazon --filter "manifest-location=*pv-grub-hd0_1.04-$arch*" --region "$region" | cut -f 2) |
||||
[ -n "$kernel" ] |
||||
echo "using PV-GRUB kernel $kernel" |
||||
extraFlags+=" --virtualization-type paravirtual --kernel $kernel" |
||||
else |
||||
extraFlags+=" --virtualization-type hvm" |
||||
fi |
||||
|
||||
set -x |
||||
ami=$(ec2-register \ |
||||
-n "$name" \ |
||||
-d "$description" \ |
||||
--region "$region" \ |
||||
--architecture "$arch" \ |
||||
$extraFlags | cut -f 2) |
||||
fi |
||||
|
||||
echo -n "$ami" > $amiFile |
||||
echo "created AMI $ami of type '$type' in $region..." |
||||
|
||||
else |
||||
ami=$(cat $amiFile) |
||||
fi |
||||
|
||||
echo "waiting for AMI..." |
||||
while true; do |
||||
status=$(ec2-describe-images "$ami" --region "$region" | head -n1 | cut -f 5) |
||||
if [ "$status" = available ]; then break; fi |
||||
sleep 10 |
||||
done |
||||
|
||||
ec2-modify-image-attribute \ |
||||
--region "$region" "$ami" -l -a all |
||||
|
||||
echo "region = $region, type = $type, store = $store, ami = $ami" |
||||
if [ -z "$prevAmi" ]; then |
||||
prevAmi="$ami" |
||||
prevRegion="$region" |
||||
fi |
||||
|
||||
echo " \"15.09\".$region.$type-$store = \"$ami\";" >> ec2-amis.nix |
||||
done |
||||
|
||||
done |
||||
|
||||
done |
||||
@ -1,216 +0,0 @@ |
||||
#! /usr/bin/env python |
||||
|
||||
import os |
||||
import sys |
||||
import time |
||||
import argparse |
||||
import nixops.util |
||||
from nixops import deployment |
||||
from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType |
||||
import boto.ec2 |
||||
from nixops.statefile import StateFile, get_default_state_file |
||||
|
||||
parser = argparse.ArgumentParser(description='Create an EBS-backed NixOS AMI') |
||||
parser.add_argument('--region', dest='region', required=True, help='EC2 region to create the image in') |
||||
parser.add_argument('--channel', dest='channel', default="14.12", help='Channel to use') |
||||
parser.add_argument('--keep', dest='keep', action='store_true', help='Keep NixOps machine after use') |
||||
parser.add_argument('--hvm', dest='hvm', action='store_true', help='Create HVM image') |
||||
parser.add_argument('--key', dest='key_name', action='store_true', help='Keypair used for HVM instance creation', default="rob") |
||||
args = parser.parse_args() |
||||
|
||||
instance_type = "m3.medium" if args.hvm else "m1.small" |
||||
|
||||
if args.hvm: |
||||
virtualization_type = "hvm" |
||||
root_block = "/dev/sda1" |
||||
image_type = 'hvm' |
||||
else: |
||||
virtualization_type = "paravirtual" |
||||
root_block = "/dev/sda" |
||||
image_type = 'ebs' |
||||
|
||||
ebs_size = 20 |
||||
|
||||
# Start a NixOS machine in the given region. |
||||
f = open("ebs-creator-config.nix", "w") |
||||
f.write('''{{ |
||||
resources.ec2KeyPairs.keypair.accessKeyId = "lb-nixos"; |
||||
resources.ec2KeyPairs.keypair.region = "{0}"; |
||||
|
||||
machine = |
||||
{{ pkgs, ... }}: |
||||
{{ |
||||
deployment.ec2.accessKeyId = "lb-nixos"; |
||||
deployment.ec2.region = "{0}"; |
||||
deployment.ec2.blockDeviceMapping."/dev/xvdg".size = pkgs.lib.mkOverride 10 {1}; |
||||
}}; |
||||
}} |
||||
'''.format(args.region, ebs_size)) |
||||
f.close() |
||||
|
||||
db = StateFile(get_default_state_file()) |
||||
try: |
||||
depl = db.open_deployment("ebs-creator") |
||||
except Exception: |
||||
depl = db.create_deployment() |
||||
depl.name = "ebs-creator" |
||||
depl.logger.set_autoresponse("y") |
||||
depl.nix_exprs = [os.path.abspath("./ebs-creator.nix"), os.path.abspath("./ebs-creator-config.nix")] |
||||
if not args.keep: depl.destroy_resources() |
||||
depl.deploy(allow_reboot=True) |
||||
|
||||
m = depl.machines['machine'] |
||||
|
||||
# Do the installation. |
||||
device="/dev/xvdg" |
||||
if args.hvm: |
||||
m.run_command('parted -s /dev/xvdg -- mklabel msdos') |
||||
m.run_command('parted -s /dev/xvdg -- mkpart primary ext2 1M -1s') |
||||
device="/dev/xvdg1" |
||||
|
||||
m.run_command("if mountpoint -q /mnt; then umount /mnt; fi") |
||||
m.run_command("mkfs.ext4 -L nixos {0}".format(device)) |
||||
m.run_command("mkdir -p /mnt") |
||||
m.run_command("mount {0} /mnt".format(device)) |
||||
m.run_command("touch /mnt/.ebs") |
||||
m.run_command("mkdir -p /mnt/etc/nixos") |
||||
|
||||
m.run_command("nix-channel --add https://nixos.org/channels/nixos-{} nixos".format(args.channel)) |
||||
m.run_command("nix-channel --update") |
||||
|
||||
version = m.run_command("nix-instantiate --eval-only -A lib.nixpkgsVersion '<nixpkgs>'", capture_stdout=True).split(' ')[0].replace('"','').strip() |
||||
print >> sys.stderr, "NixOS version is {0}".format(version) |
||||
if args.hvm: |
||||
m.upload_file("./amazon-base-config.nix", "/mnt/etc/nixos/amazon-base-config.nix") |
||||
m.upload_file("./amazon-hvm-config.nix", "/mnt/etc/nixos/configuration.nix") |
||||
m.upload_file("./amazon-hvm-install-config.nix", "/mnt/etc/nixos/amazon-hvm-install-config.nix") |
||||
m.run_command("NIXOS_CONFIG=/etc/nixos/amazon-hvm-install-config.nix nixos-install") |
||||
else: |
||||
m.upload_file("./amazon-base-config.nix", "/mnt/etc/nixos/configuration.nix") |
||||
m.run_command("nixos-install") |
||||
|
||||
m.run_command("umount /mnt") |
||||
|
||||
if args.hvm: |
||||
ami_name = "nixos-{0}-x86_64-hvm".format(version) |
||||
description = "NixOS {0} (x86_64; EBS root; hvm)".format(version) |
||||
else: |
||||
ami_name = "nixos-{0}-x86_64-ebs".format(version) |
||||
description = "NixOS {0} (x86_64; EBS root)".format(version) |
||||
|
||||
|
||||
# Wait for the snapshot to finish. |
||||
def check(): |
||||
status = snapshot.update() |
||||
print >> sys.stderr, "snapshot status is {0}".format(status) |
||||
return status == '100%' |
||||
|
||||
m.connect() |
||||
volume = m._conn.get_all_volumes([], filters={'attachment.instance-id': m.resource_id, 'attachment.device': "/dev/sdg"})[0] |
||||
|
||||
# Create a snapshot. |
||||
snapshot = volume.create_snapshot(description=description) |
||||
print >> sys.stderr, "created snapshot {0}".format(snapshot.id) |
||||
|
||||
nixops.util.check_wait(check, max_tries=120) |
||||
|
||||
m._conn.create_tags([snapshot.id], {'Name': ami_name}) |
||||
|
||||
if not args.keep: depl.destroy_resources() |
||||
|
||||
# Register the image. |
||||
aki = m._conn.get_all_images(filters={'manifest-location': 'ec2*pv-grub-hd0_1.03-x86_64*'})[0] |
||||
print >> sys.stderr, "using kernel image {0} - {1}".format(aki.id, aki.location) |
||||
|
||||
block_map = BlockDeviceMapping() |
||||
block_map[root_block] = BlockDeviceType(snapshot_id=snapshot.id, delete_on_termination=True, size=ebs_size, volume_type="gp2") |
||||
block_map['/dev/sdb'] = BlockDeviceType(ephemeral_name="ephemeral0") |
||||
block_map['/dev/sdc'] = BlockDeviceType(ephemeral_name="ephemeral1") |
||||
block_map['/dev/sdd'] = BlockDeviceType(ephemeral_name="ephemeral2") |
||||
block_map['/dev/sde'] = BlockDeviceType(ephemeral_name="ephemeral3") |
||||
|
||||
common_args = dict( |
||||
name=ami_name, |
||||
description=description, |
||||
architecture="x86_64", |
||||
root_device_name=root_block, |
||||
block_device_map=block_map, |
||||
virtualization_type=virtualization_type, |
||||
delete_root_volume_on_termination=True |
||||
) |
||||
if not args.hvm: |
||||
common_args['kernel_id']=aki.id |
||||
|
||||
ami_id = m._conn.register_image(**common_args) |
||||
|
||||
print >> sys.stderr, "registered AMI {0}".format(ami_id) |
||||
|
||||
print >> sys.stderr, "sleeping a bit..." |
||||
time.sleep(30) |
||||
|
||||
print >> sys.stderr, "setting image name..." |
||||
m._conn.create_tags([ami_id], {'Name': ami_name}) |
||||
|
||||
print >> sys.stderr, "making image public..." |
||||
image = m._conn.get_all_images(image_ids=[ami_id])[0] |
||||
image.set_launch_permissions(user_ids=[], group_names=["all"]) |
||||
|
||||
# Do a test deployment to make sure that the AMI works. |
||||
f = open("ebs-test.nix", "w") |
||||
f.write( |
||||
''' |
||||
{{ |
||||
network.description = "NixOS EBS test"; |
||||
|
||||
resources.ec2KeyPairs.keypair.accessKeyId = "lb-nixos"; |
||||
resources.ec2KeyPairs.keypair.region = "{0}"; |
||||
|
||||
machine = {{ config, pkgs, resources, ... }}: {{ |
||||
deployment.targetEnv = "ec2"; |
||||
deployment.ec2.accessKeyId = "lb-nixos"; |
||||
deployment.ec2.region = "{0}"; |
||||
deployment.ec2.instanceType = "{2}"; |
||||
deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name; |
||||
deployment.ec2.securityGroups = [ "public-ssh" ]; |
||||
deployment.ec2.ami = "{1}"; |
||||
}}; |
||||
}} |
||||
'''.format(args.region, ami_id, instance_type)) |
||||
f.close() |
||||
|
||||
test_depl = db.create_deployment() |
||||
test_depl.auto_response = "y" |
||||
test_depl.name = "ebs-creator-test" |
||||
test_depl.nix_exprs = [os.path.abspath("./ebs-test.nix")] |
||||
test_depl.deploy(create_only=True) |
||||
test_depl.machines['machine'].run_command("nixos-version") |
||||
|
||||
# Log the AMI ID. |
||||
f = open("ec2-amis.nix".format(args.region, image_type), "w") |
||||
f.write("{\n") |
||||
|
||||
for dest in [ 'us-east-1', 'us-west-1', 'us-west-2', 'eu-west-1', 'eu-central-1', 'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'sa-east-1']: |
||||
copy_image = None |
||||
if args.region != dest: |
||||
try: |
||||
print >> sys.stderr, "copying image from region {0} to {1}".format(args.region, dest) |
||||
conn = boto.ec2.connect_to_region(dest) |
||||
copy_image = conn.copy_image(args.region, ami_id, ami_name, description=None, client_token=None) |
||||
except : |
||||
print >> sys.stderr, "FAILED!" |
||||
|
||||
# Log the AMI ID. |
||||
if copy_image != None: |
||||
f.write(' "{0}"."{1}".{2} = "{3}";\n'.format(args.channel,dest,"hvm" if args.hvm else "ebs",copy_image.image_id)) |
||||
else: |
||||
f.write(' "{0}"."{1}".{2} = "{3}";\n'.format(args.channel,args.region,"hvm" if args.hvm else "ebs",ami_id)) |
||||
|
||||
|
||||
f.write("}\n") |
||||
f.close() |
||||
|
||||
if not args.keep: |
||||
test_depl.logger.set_autoresponse("y") |
||||
test_depl.destroy_resources() |
||||
test_depl.delete() |
||||
|
||||
@ -1,53 +0,0 @@ |
||||
#! /bin/sh -e |
||||
|
||||
export NIXOS_CONFIG=$(dirname $(readlink -f $0))/amazon-base-config.nix |
||||
|
||||
version=$(nix-instantiate --eval-only '<nixpkgs/nixos>' -A config.system.nixosVersion | sed s/'"'//g) |
||||
echo "NixOS version is $version" |
||||
|
||||
buildAndUploadFor() { |
||||
system="$1" |
||||
arch="$2" |
||||
|
||||
echo "building $system image..." |
||||
nix-build '<nixpkgs/nixos>' \ |
||||
-A config.system.build.amazonImage --argstr system "$system" -o ec2-ami |
||||
|
||||
ec2-bundle-image -i ./ec2-ami/nixos.img --user "$AWS_ACCOUNT" --arch "$arch" \ |
||||
-c "$EC2_CERT" -k "$EC2_PRIVATE_KEY" |
||||
|
||||
for region in eu-west-1; do |
||||
echo "uploading $system image for $region..." |
||||
|
||||
name=nixos-$version-$arch-s3 |
||||
bucket="$(echo $name-$region | tr '[A-Z]_' '[a-z]-')" |
||||
|
||||
if [ "$region" = eu-west-1 ]; then s3location=EU; |
||||
elif [ "$region" = us-east-1 ]; then s3location=US; |
||||
else s3location="$region" |
||||
fi |
||||
|
||||
ec2-upload-bundle -b "$bucket" -m /tmp/nixos.img.manifest.xml \ |
||||
-a "$EC2_ACCESS_KEY" -s "$EC2_SECRET_KEY" --location "$s3location" \ |
||||
--url http://s3.amazonaws.com |
||||
|
||||
kernel=$(ec2-describe-images -o amazon --filter "manifest-location=*pv-grub-hd0_1.04-$arch*" --region "$region" | cut -f 2) |
||||
echo "using PV-GRUB kernel $kernel" |
||||
|
||||
ami=$(ec2-register "$bucket/nixos.img.manifest.xml" -n "$name" -d "NixOS $system r$revision" -O "$EC2_ACCESS_KEY" -W "$EC2_SECRET_KEY" \ |
||||
--region "$region" --kernel "$kernel" | cut -f 2) |
||||
|
||||
echo "AMI ID is $ami" |
||||
|
||||
echo " \"14.12\".\"$region\".s3 = \"$ami\";" >> ec2-amis.nix |
||||
|
||||
ec2-modify-image-attribute --region "$region" "$ami" -l -a all -O "$EC2_ACCESS_KEY" -W "$EC2_SECRET_KEY" |
||||
|
||||
for cp_region in us-east-1 us-west-1 us-west-2 eu-central-1 ap-southeast-1 ap-southeast-2 ap-northeast-1 sa-east-1; do |
||||
new_ami=$(aws ec2 copy-image --source-image-id $ami --source-region $region --region $cp_region --name "$name" | json ImageId) |
||||
echo " \"14.12\".\"$cp_region\".s3 = \"$new_ami\";" >> ec2-amis.nix |
||||
done |
||||
done |
||||
} |
||||
|
||||
buildAndUploadFor x86_64-linux x86_64 |
||||
@ -1,13 +0,0 @@ |
||||
{ |
||||
network.description = "NixOS EBS creator"; |
||||
|
||||
machine = |
||||
{ config, pkgs, resources, ... }: |
||||
{ deployment.targetEnv = "ec2"; |
||||
deployment.ec2.instanceType = "c3.large"; |
||||
deployment.ec2.securityGroups = [ "public-ssh" ]; |
||||
deployment.ec2.ebsBoot = false; |
||||
deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name; |
||||
environment.systemPackages = [ pkgs.parted ]; |
||||
}; |
||||
} |
||||
@ -1,3 +0,0 @@ |
||||
{ |
||||
imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ]; |
||||
} |
||||
@ -0,0 +1,47 @@ |
||||
# This module automatically grows the root partition on Amazon EC2 HVM |
||||
# instances. This allows an instance to be created with a bigger root |
||||
# filesystem than provided by the AMI. |
||||
|
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
|
||||
growpart = pkgs.stdenv.mkDerivation { |
||||
name = "growpart"; |
||||
src = pkgs.fetchurl { |
||||
url = "https://launchpad.net/cloud-utils/trunk/0.27/+download/cloud-utils-0.27.tar.gz"; |
||||
sha256 = "16shlmg36lidp614km41y6qk3xccil02f5n3r4wf6d1zr5n4v8vd"; |
||||
}; |
||||
patches = [ ./growpart-util-linux-2.26.patch ]; |
||||
buildPhase = '' |
||||
cp bin/growpart $out |
||||
sed -i 's|awk|gawk|' $out |
||||
sed -i 's|sed|gnused|' $out |
||||
''; |
||||
dontInstall = true; |
||||
dontPatchShebangs = true; |
||||
}; |
||||
|
||||
in |
||||
|
||||
{ |
||||
|
||||
config = mkIf config.ec2.hvm { |
||||
|
||||
boot.initrd.extraUtilsCommands = '' |
||||
copy_bin_and_libs ${pkgs.gawk}/bin/gawk |
||||
copy_bin_and_libs ${pkgs.gnused}/bin/sed |
||||
copy_bin_and_libs ${pkgs.utillinux}/sbin/sfdisk |
||||
cp -v ${growpart} $out/bin/growpart |
||||
ln -s sed $out/bin/gnused |
||||
''; |
||||
|
||||
boot.initrd.postDeviceCommands = '' |
||||
[ -e /dev/xvda ] && [ -e /dev/xvda1 ] && TMPDIR=/run sh $(type -P growpart) /dev/xvda 1 |
||||
''; |
||||
|
||||
}; |
||||
|
||||
} |
||||
@ -0,0 +1,88 @@ |
||||
From 1895d10a7539d055a4e0206af1e7a9e5ea32a4f7 Mon Sep 17 00:00:00 2001
|
||||
From: Juerg Haefliger <juerg.haefliger@hp.com>
|
||||
Date: Wed, 25 Mar 2015 13:59:20 +0100
|
||||
Subject: [PATCH] Support new sfdisk version 2.26
|
||||
|
||||
The sfdisk usage with version 2.26 changed. Specifically, the option
|
||||
--show-pt-geometry and functionality for CHS have been removed.
|
||||
Also, restoring a backup MBR now needs to be done using dd.
|
||||
---
|
||||
bin/growpart | 28 ++++++++++------------------
|
||||
1 file changed, 10 insertions(+), 18 deletions(-)
|
||||
|
||||
diff --git a/bin/growpart b/bin/growpart
|
||||
index 595c40b..d4c995b 100755
|
||||
--- a/bin/growpart
|
||||
+++ b/bin/growpart
|
||||
@@ -28,7 +28,6 @@ PART=""
|
||||
PT_UPDATE=false
|
||||
DRY_RUN=0
|
||||
|
||||
-MBR_CHS=""
|
||||
MBR_BACKUP=""
|
||||
GPT_BACKUP=""
|
||||
_capture=""
|
||||
@@ -133,7 +132,8 @@ bad_Usage() {
|
||||
}
|
||||
|
||||
mbr_restore() {
|
||||
- sfdisk --no-reread "${DISK}" ${MBR_CHS} -I "${MBR_BACKUP}"
|
||||
+ dd if="${MBR_BACKUP}-${DISK#/dev/}-0x00000000.bak" of="${DISK}" bs=1 \
|
||||
+ conv=notrunc
|
||||
}
|
||||
|
||||
sfdisk_worked_but_blkrrpart_failed() {
|
||||
@@ -148,34 +148,26 @@ sfdisk_worked_but_blkrrpart_failed() {
|
||||
|
||||
mbr_resize() {
|
||||
RESTORE_HUMAN="${TEMP_D}/recovery"
|
||||
- MBR_BACKUP="${TEMP_D}/orig.save"
|
||||
+ MBR_BACKUP="${TEMP_D}/backup"
|
||||
|
||||
local change_out=${TEMP_D}/change.out
|
||||
local dump_out=${TEMP_D}/dump.out
|
||||
local new_out=${TEMP_D}/new.out
|
||||
local dump_mod=${TEMP_D}/dump.mod
|
||||
- local tmp="${TEMP_D}/tmp.out"
|
||||
- local err="${TEMP_D}/err.out"
|
||||
|
||||
- local _devc cyl _w1 heads _w2 sectors _w3 tot dpart
|
||||
+ local tot dpart
|
||||
local pt_start pt_size pt_end max_end new_size change_info
|
||||
|
||||
- # --show-pt-geometry outputs something like
|
||||
- # /dev/sda: 164352 cylinders, 4 heads, 32 sectors/track
|
||||
- rqe sfd_geom sfdisk "${DISK}" --show-pt-geometry >"${tmp}" &&
|
||||
- read _devc cyl _w1 heads _w2 sectors _w3 <"${tmp}" &&
|
||||
- MBR_CHS="-C ${cyl} -H ${heads} -S ${sectors}" ||
|
||||
- fail "failed to get CHS from ${DISK}"
|
||||
+ tot=$(sfdisk --list "${DISK}" | awk '{ print $(NF-1) ; exit }') ||
|
||||
+ fail "failed to get total number of sectors from ${DISK}"
|
||||
|
||||
- tot=$((${cyl}*${heads}*${sectors}))
|
||||
+ debug 1 "total number of sectors of ${DISK} is ${tot}"
|
||||
|
||||
- debug 1 "geometry is ${MBR_CHS}. total size=${tot}"
|
||||
- rqe sfd_dump sfdisk ${MBR_CHS} --unit=S --dump "${DISK}" \
|
||||
+ rqe sfd_dump sfdisk --dump "${DISK}" \
|
||||
>"${dump_out}" ||
|
||||
fail "failed to dump sfdisk info for ${DISK}"
|
||||
-
|
||||
{
|
||||
- echo "## sfdisk ${MBR_CHS} --unit=S --dump ${DISK}"
|
||||
+ echo "## sfdisk --dump ${DISK}"
|
||||
cat "${dump_out}"
|
||||
} >"${RESTORE_HUMAN}"
|
||||
[ $? -eq 0 ] || fail "failed to save sfdisk -d output"
|
||||
@@ -237,7 +229,7 @@ mbr_resize() {
|
||||
exit 0
|
||||
fi
|
||||
|
||||
- LANG=C sfdisk --no-reread "${DISK}" ${MBR_CHS} --force \
|
||||
+ LANG=C sfdisk --no-reread "${DISK}" --force \
|
||||
-O "${MBR_BACKUP}" <"${new_out}" >"${change_out}" 2>&1
|
||||
ret=$?
|
||||
[ $ret -eq 0 ] || RESTORE_FUNC="mbr_restore"
|
||||
--
|
||||
2.1.4
|
||||
|
||||
Loading…
Reference in new issue