From b4e94e5ffeb834f96c5f3338c36b22fc13bd0cee Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Wed, 19 May 2021 18:38:50 +0200 Subject: [PATCH 1/2] Add stdenv_prefix --- stdenv_prefix/example.nix | 51 +++++++++++++++ stdenv_prefix/prehook.sh | 98 ++++++++++++++++++++++++++++ stdenv_prefix/stdenv.md | 133 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 stdenv_prefix/example.nix create mode 100644 stdenv_prefix/prehook.sh create mode 100644 stdenv_prefix/stdenv.md diff --git a/stdenv_prefix/example.nix b/stdenv_prefix/example.nix new file mode 100644 index 0000000..0d4aa65 --- /dev/null +++ b/stdenv_prefix/example.nix @@ -0,0 +1,51 @@ +{ storepath_length ? 30, storepath_chunks ? 4, pkgs ? import { + stdenvStages = { config, overlays, localSystem, ... }@args: (import args) ++ [ + (previousStage: { + inherit config overlays; + stdenv = let + toPreHook = withAppendOut: '' + storepath_length=${toString storepath_length} + storepath_chunks=${toString storepath_chunks} + '' + builtins.readFile ./prehook.sh + (pkgs.lib.optionalString withAppendOut '' + appendOut + ''); + newlibunistring = previousStage.libunistring.overrideAttrs (attrs: { + preHook = attrs.preHook or "" + (toPreHook true); + am_cv_func_iconv_works = "yes"; + }); + newlibidn2 = (previousStage.libidn2.override { libunistring = newlibunistring; }).overrideAttrs (attrs: { + preHook = attrs.preHook or "" + (toPreHook true); + postFixup = '' + ${previousStage.nukeReferences}/bin/nuke-refs -e $(cat ${newlibunistring}/nix-support/new-out)/lib \ + "$out"/lib/lib*.so.*.* + ''; + }); + overridenCallPackage = p: a: previousStage.callPackage p (a // { libidn2 = newlibidn2;}); + + newlibc = (previousStage."${localSystem.libc}".override { callPackage = overridenCallPackage; }).overrideAttrs(old: { + preHook = old.preHook or "" + (toPreHook true); + }); + newbintools = (previousStage.binutils.override { libc = newlibc; }).overrideAttrs(old: { + postFixup = old.postFixup + '' + newlibcout=$(cat ${newlibc}/nix-support/new-out) + sed -i -e "s@${newlibc}@$newlibcout@g" $out/nix-support/* + ''; + }); + newStdenv = previousStage.stdenv.override { + allowedRequisites = null; + extraAttrs.bintools = newbintools; + cc = previousStage.gcc.override({ bintools = newbintools; libc = newlibc; }); + overrides = self: super: rewriteMap (previousStage.stdenv.overrides self super) // { + ${localSystem.libc} = newlibc; + }; + preHook = previousStage.stdenv.preHook + '' + libc_path=${newlibc} + '' + (toPreHook false); + }; + rewriteMap = builtins.mapAttrs (n: v: v.override { stdenv = newStdenv; }); + in + newStdenv; + }) + ]; +} }: + pkgs.xar diff --git a/stdenv_prefix/prehook.sh b/stdenv_prefix/prehook.sh new file mode 100644 index 0000000..db9a4d2 --- /dev/null +++ b/stdenv_prefix/prehook.sh @@ -0,0 +1,98 @@ +count_chunks() { + elements="${1//[^\/]}" + echo $(( "${#elements}" )) +} + +oldout=$out +prefix_chunks=$(count_chunks $NIX_STORE) + +: ${storepath_length=${#NIX_STORE}} +: ${storepath_chunks=$prefix_chunks} + +if [ "$storepath_chunks" -lt $prefix_chunks ]; then + echo "Need to have at least as much path elements as the initial prefix" + exit 1 +fi + +min_length=$(( ${#NIX_STORE} + 2 * $storepath_chunks - 2 * $prefix_chunks )) +if [ "$storepath_length" -lt "$min_length" ]; then + echo "The storepath length needs to be at least the length of the prefix" + exit 1 +fi + +if [ "$storepath_chunks" -eq "$prefix_chunks" -a "$storepath_length" -gt "$min_length" ]; then + echo "You need at least one more chunk for a non-standard path length" + exit 1 +fi + +appendOut() { + chunk_prefix=$(for (( c=$prefix_chunks ; c < $storepath_chunks ; c++ )); do echo -n "/x"; done) + length_prefix=$(for (( c=$(( ${#NIX_STORE} + ${#chunk_prefix} )) ; c < $storepath_length ; c++ )); do echo -n "x"; done) + out="$out$chunk_prefix$length_prefix" + if [ -n $chunk_prefix -o -n $length_prefix ]; then + mkdir -p $(dirname $out) + fi + + : ${libc_path=} + : ${configureFlags=} + configureFlags="${configureFlags//$oldout/$out}" + if [ -n "$libc_path" ]; then + libc_oldpath=$libc_path/lib + libc_newpath=$(cat $libc_path/nix-support/new-out)/lib + configureFlags="${configureFlags//$libc_oldpath/$libc_newpath}" + fi +} + +addNixSupport() { + if [ ! -d $out ]; then + return + fi + mkdir -p $out/nix-support + echo $out > $out/nix-support/new-out +} + +overridePlaceholder() { +function _callImplicitHook { + local def="$1" + local hookName="$2" + if declare -F "$hookName" > /dev/null; then + "$hookName" + elif type -p "$hookName" > /dev/null; then + source "$hookName" + elif [ -n "${!hookName:-}" ]; then + content="${!hookName}" + eval "${content//$oldout/$out}" + else + return "$def" + fi +} +} + +preHooks+=(appendOut overridePlaceholder) + +restoreSingleFile() { + if [ "$out" != "$oldout" -a -f $out ]; then + cp -a $out ./file + rm -rf $oldout + cp -a ./file $oldout + fi +} + +addSymlinks() { + if [ "$out" != "$oldout" -a -d $out ]; then + ln -s $out/* $oldout/ + fi +} + +rename_generic_build() { +eval "$(echo "orig_genericBuild()"; declare -f genericBuild | tail -n +2)" +function genericBuild { + orig_genericBuild "$@" + restoreSingleFile + addSymlinks + addNixSupport +} +} + +exitHooks+=(restoreSingleFile addSymlinks addNixSupport) +postHooks+=(rename_generic_build) diff --git a/stdenv_prefix/stdenv.md b/stdenv_prefix/stdenv.md new file mode 100644 index 0000000..f58455a --- /dev/null +++ b/stdenv_prefix/stdenv.md @@ -0,0 +1,133 @@ +Ce dossier contient un travail permettant de créer des dérivations qui +vont toutes commencer par un préfix de taille et avec un nombre de +composants donné (de la forme x/x/xxxxx) + +Le but étant de pouvoir copier une closure d’une dérivation, la mettre +par exemple dans /opt/nix/store/....-foo et avec un simple `rename` + +`sed` la rendre utilisable (dans l’exemple /nix/store/....-foo/xxx -> +/opt/nix/store/....-foo). + +Ci dessous se trouvent des notes de ce que j’ai compris de la stack +stdenv (pkgs/stdenv/linux/default.nix) + +# Stage -1 + gcc-unwrapped = null + binutils = null + coreutils = null + gnugrep = null + `stagem1 = stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages` + -> contient cinq entrées (dont stdenv "invalide") + +# Stage 0 + `stage0 = stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages` + (`stagem1 = stage0.stdenv.__bootPackages`) + `stage0.stdenv` + -> cc = null + `stage0` + overrides: + - glibc = extrait (ln -s) de bootstrapTools (bootstrap-stage0-glibc) + - gcc-unwrapped = bootstrapTools + - binutils = wrapper sur bootstrapTools (bootstrap-stage0-binutils-wrapper) + - coreutils = bootstrapTools + - gnugrep = bootstrapTools + - ccWrapperStdenv = stage0.stdenv + +# Stage 1 + `stage1 = stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages` + (`stage0 = stage1.stdenv.__bootPackages`) + `stage1.stdenv` + -> cc = cc-wrapper buildé avec + cc = stage0.gcc-unwrapped = bootstrapTools + bintools = stage0.binutils = bootstrap-stage0-binutils-wrapper + glibc = stage0.glibc = bootstrap-stage0-glibc + coreutils = stage0.coreutils = bootstrapTools + gnugrep = stage0.gnugrep = bootstrapTools + stdenvNoCC = stage0.ccWrapperStdenv = stage0.stdenv + `stage1` + overrides: + - binutils-unwrapped = override (gold = false) + - ccWrapperStdenv = stage0.stdenv + - gcc-unwrapped = bootstrapTools + - coreutils = bootstrapTools + - gnugrep = bootstrapTools + - glibc = bootstrap-stage0-glibc + - perl = override (threading = false) + +# Stage 2 + `stage2 = stdenv.__bootPackages.stdenv.__bootPackages.stdenv.__bootPackages` + (`stage1 = stage2.stdenv.__bootPackages`) + `stage2.stdenv` + -> cc = cc-wrapper buildé avec + cc = stage1.gcc-unwrapped = bootstrapTools + bintools = stage1.binutils = wrapper sur binutils-unwrapped (gold = false) + glibc = bootstrap-stage0-glibc + coreutils = bootstrapTools + gnugrep = bootstrapTools + stdenvNoCC = stage1.ccWrapperStdenv = stage0.stdenv + `stage2` + overrides: + - ccWrapperStdenv = stage0.stdenv + - gcc-unwrapped = bootstrapTools + - coreutils = bootstrapTools + - gnugrep = bootstrapTools + - perl = override (threading = false) + - binutils = stage1.override (libc = self.glibc) + - libidn2 = nuke-refs + - libunistring = nuke-refs + - dejagnu = nocheck + - gnum4 = stage1.gnum4 + - bison = stage1.bison + +# Stage 3 + `stage3 = stdenv.__bootPackages.stdenv.__bootPackages` + (`stage2 = stage3.stdenv.__bootPackages`) + `stage3.stdenv` + -> cc = cc-wrapper buildé avec + cc = stage2.gcc-unwrapped = bootstrapTools + bintools = stage2.binutils + glibc = stage2.glibc + coreutils = bootstrapTools + gnugrep = bootstrapTools + stdenvNoCC = stage2.ccWrapperStdenv = stage0.stdenv + `stage3` + overrides: + - ccWrapperStdenv = stage0.stdenv + - binutils = stage2.binutils + - coreutils = bootstrapTools + - gnugrep = bootstrapTools + - perl patchelf linuxHeaders gnum4 bison libidn2 libunistring = stage2 + - glibc = stage2.glibc + - gmp mpfr libmpc isl_0_20 = override static + - gcc-unwrapped = override (isl) + +# Stage 4 + `stage4 = stdenv.__bootPackages` + (`stage3 = stage4.stdenv.__bootPackages`) + `stage4.stdenv` + -> cc = cc-wrapper buildé avec + cc = stage3.gcc-unwrapped = override (isl) + bintools = stage2.binutils + glibc = stage2.glibc + coreutils = bootstrapTools + gnugrep = bootstrapTools + stdenvNoCC = stage3.ccWrapperStdenv = stage0.stdenv + `stage4` + overrides: + - gettext gnum4 bison gmp perl texinfo zlib linuxHeaders libidn2 libunistring = stage3 + - glibc = stage3.glibc = stage2.glibc + - binutils = override + - gcc = buildé avec + cc = stage3.gcc-unwrapped + bintools = self.binutils + libc = self.glibc + stdenvNoCC coreutils gnugrep = self + +# Final Stage + `stdenv` + cc = stage4.gcc + `pkgs` + overrides: + - glibc = stage4.glibc = stage2.glibc + - zlib libidn2 libunistring = stage4 = stage3 + - (hors cross-compile) binutils binutils-unwrapped = stage4 + - (hors cross-compile) gcc = cc -- 2.41.0 From 5275f756ad115bbcacce52eb04eeae4a303e4b2e Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Sun, 27 Jun 2021 01:48:12 +0200 Subject: [PATCH 2/2] Add some fixes to stdenv prefix hook --- stdenv_prefix/example.nix | 34 ++++++++++++++++++++++++++++++++-- stdenv_prefix/prehook.sh | 3 +++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/stdenv_prefix/example.nix b/stdenv_prefix/example.nix index 0d4aa65..be66db7 100644 --- a/stdenv_prefix/example.nix +++ b/stdenv_prefix/example.nix @@ -1,8 +1,29 @@ -{ storepath_length ? 30, storepath_chunks ? 4, pkgs ? import { +{ storepath_length ? 42, storepath_chunks ? 6, pkgs ? import { + overlays = [ + (self: super: { + zstd = super.zstd.overrideAttrs (old: { + postPatch = old.postPatch + '' + sed -i 149d build/cmake/lib/CMakeLists.txt + ''; + }); + glibcLocales = super.glibcLocales.overrideAttrs (old: let + libc = self.buildPackages.stdenv.cc.libc; + in { + buildPhase = '' + mkdir -vp "$TMPDIR/${libc.new-out}/lib/locale" + '' + old.buildPhase; + installPhase = '' + rmdir $TMPDIR/"${libc.out}/lib/locale" + ln -s "$TMPDIR/${libc.new-out}/lib/locale" $TMPDIR/"${libc.out}/lib/locale" + '' + old.installPhase; + }); + }) + ]; stdenvStages = { config, overlays, localSystem, ... }@args: (import args) ++ [ (previousStage: { inherit config overlays; stdenv = let + toNewOut = drv: builtins.readFile (pkgs.runCommand "new-out" {} "echo -n $(cat ${drv}/nix-support/new-out) > $out"); toPreHook = withAppendOut: '' storepath_length=${toString storepath_length} storepath_chunks=${toString storepath_chunks} @@ -24,6 +45,7 @@ newlibc = (previousStage."${localSystem.libc}".override { callPackage = overridenCallPackage; }).overrideAttrs(old: { preHook = old.preHook or "" + (toPreHook true); + passthru = { new-out = toNewOut newlibc; }; }); newbintools = (previousStage.binutils.override { libc = newlibc; }).overrideAttrs(old: { postFixup = old.postFixup + '' @@ -36,6 +58,7 @@ extraAttrs.bintools = newbintools; cc = previousStage.gcc.override({ bintools = newbintools; libc = newlibc; }); overrides = self: super: rewriteMap (previousStage.stdenv.overrides self super) // { + bash = super.bash.overrideAttrs (old: { passthru = old.passthru // { new-out = toNewOut super.bash; }; }); ${localSystem.libc} = newlibc; }; preHook = previousStage.stdenv.preHook + '' @@ -46,6 +69,13 @@ in newStdenv; }) + (previousStage: { + inherit config overlays; + stdenv = previousStage.stdenv.override { + shell = previousStage.bash + "/bin/bash"; + initialPath = ((import ) {pkgs = previousStage;}); + }; + }) ]; } }: - pkgs.xar + pkgs diff --git a/stdenv_prefix/prehook.sh b/stdenv_prefix/prehook.sh index db9a4d2..c5474dc 100644 --- a/stdenv_prefix/prehook.sh +++ b/stdenv_prefix/prehook.sh @@ -35,11 +35,14 @@ appendOut() { : ${libc_path=} : ${configureFlags=} + : ${installFlags=} + installFlags="${installFlags//$oldout/$out}" configureFlags="${configureFlags//$oldout/$out}" if [ -n "$libc_path" ]; then libc_oldpath=$libc_path/lib libc_newpath=$(cat $libc_path/nix-support/new-out)/lib configureFlags="${configureFlags//$libc_oldpath/$libc_newpath}" + installFlags="${installFlags//$libc_oldpath/$libc_newpath}" fi } -- 2.41.0