Faster, more succeeding builds, better everything

- Now fetching nixpgks with fetchFromGitHub by computing and saving the
sha256 at update time
- Detecting core GHC libraries and setting them to null to fix builds,
workaround for input-output-hk/stack2nix#134
- Add override directory to fix builds for specific ghc versions
- Speed up builds by disabling library profiling for everything
- New nix file structure with powerful selection mechanisms

Author: infinisil

.gitignore

@@ -0,0 +1 @@
+result*

Design.org

@@ -5,21 +5,31 @@ Assumption: The stack build does not depend on which nixpkgs version it uses, it
 - history-nixpkgs-unstable
 - nixpkgs
 - haskell-ide-engine
+- per-ghcMinor
+  - <major><minor>: File with the list of base libraries for that ghc version
 - per-nixpkgs
-  - <nixpkgsrev-ghcVersions>: File with a list of ghc versions this nixpkgs rev has
+  - ghcVersions
+    - <nixpkgsrev>: File with a list of ghc versions this nixpkgs rev has
+  - sha256
+    - <nixpkgsRev>: sha256 hash for fetchFromGitHub
 - per-haskell-ide-engine
   - <hie-rev>
     - <ghcVersion-stack2nix>.nix: File generated by stack2nix for hie-rev and ghcVersion
 
-./generated/ layout:
+./ layout:
 - nixpkgsForGhc
   - <ghcVersion>: File containing the nixpkgsrev to use for that ghc
-- stable
-  - <ghcVersion>.nix: File copied from the cache
-  - revision: File with the git revision of current stable hie
-- unstable
-  - <ghcVersion>.nix: File copied from the cache
-  - revision: File with the git revision of current unstable hie
+- nixpkgsHashes
+  - <nixpkgsRevision>: File containing the sha256 hash for fetchFromGitHub for a nixpkgs version
+- ghcBaseLibraries:
+  - <ghcVersion>: List of base libraries for that ghc version
+- versions
+  - stable
+    - <ghcVersion>.nix: File copied from the cache
+    - revision: File with the git revision of current stable hie
+  - unstable
+    - <ghcVersion>.nix: File copied from the cache
+    - revision: File with the git revision of current unstable hie
 
 Flags:
 - Whether master should be regenerated

default.nix

@@ -3,36 +3,106 @@ with builtins;
 
 let
 
-  nixpkgsForGhc = mapAttrs (file: _: readFile (./nixpkgsForGhc + "/${file}"))
+
+  pkgsForGhc = mapAttrs (ghc: _: let
+    rev = readFile (./nixpkgsForGhc + "/${ghc}");
+    sha256 = readFile (./nixpkgsHashes + "/${rev}");
+  in
+    import (fetchTarball {
+      url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
+      inherit sha256;
+    }) {
+      config = {};
+      overlays = [];
+    })
     (readDir ./nixpkgsForGhc);
 
   version = name:
     mapAttrs' (file: _: let
       ghcVersion = removeSuffix ".nix" file;
-      pkgs = import (fetchGit {
-        url = "https://github.com/NixOS/nixpkgs";
-        rev = nixpkgsForGhc.${ghcVersion};
-      }) {};
-      build = pkgs.haskell.lib.justStaticExecutables
-        (import (./versions + "/${name}/${file}") {
+      pkgs = pkgsForGhc.${ghcVersion};
+      inherit (pkgs) lib haskell;
+
+      overrideFun = old: {
+        overrides = lib.composeExtensions
+          (lib.composeExtensions
+            (old.overrides or (self: super: {}))
+            (hself: hsuper: {
+
+              # Disable library profiling for faster builds
+              mkDerivation = args: hsuper.mkDerivation (args // {
+                enableLibraryProfiling = false;
+              });
+
+              # Embed the ghc version into the name
+              haskell-ide-engine = haskell.lib.overrideCabal hsuper.haskell-ide-engine (old: {
+                pname = "${old.pname}-${ghcVersion}";
+              });
+            }
+            // lib.flip genAttrs (name: null)
+              (lib.filter (name: name != "ghc" && name != "Cabal")
+                (map (name: (builtins.parseDrvName name).name)
+                  (lib.splitString " "
+                    (builtins.readFile (./ghcBaseLibraries + "/${ghcVersion}")))))
+            )
+          )
+          (if builtins.pathExists (./overrides + "/${ghcVersion}.nix") then
+            import (./overrides + "/${ghcVersion}.nix")
+            else self: super: {}
+          );
+      };
+
+      build = haskell.lib.justStaticExecutables
+        ((import (./versions + "/${name}/${file}") {
           inherit pkgs;
-        }).haskell-ide-engine;
-    in if hasSuffix ".nix" file then {
+        }).override overrideFun).haskell-ide-engine;
+    in {
       name = ghcVersion;
       value = build;
-    } else {
-      name = file;
-      value = builtins.readFile (./versions + "/${name}/${file}");
-    })
-    (readDir (./versions + "/${name}"));
-
-in rec {
+    }) 
+    (filterAttrs (file: _: file != "revision") (readDir (./versions + "/${name}")));
 
   versions = mapAttrs (file: _: version file) (readDir ./versions);
 
-  forGhc = versions.unstable // versions.stable;
+  pkgs = import <nixpkgs> {};
+  lib = pkgs.lib;
+
+  parseNixGhcVersion = version:
+    lib.concatStringsSep "." (builtins.match "ghc(.)(.)(.)" version);
+
+  combine = versions: let
+    wrapperVersion = lib.last (lib.attrNames versions);
+  in pkgs.runCommand "haskell-ide-engine-combined" {
+    nativeBuildInputs = [ pkgs.makeWrapper ];
+  } ''
+    mkdir -p $out/bin
+
+    ${concatMapStringsSep "\n" (ghcVersion: ''
+      ln -s ${versions.${ghcVersion}}/bin/hie $out/bin/hie-${parseNixGhcVersion ghcVersion}
+    '') (lib.attrNames versions)}
+
+    makeWrapper ${versions.${wrapperVersion}}/bin/hie-wrapper $out/bin/hie-wrapper \
+      --suffix PATH : $out/bin
+    ln -s hie-wrapper $out/bin/hie
+  '';
+
+  inherit (versions) stable unstable;
+
+  makeSet = versions: combine versions // {
+    inherit versions;
+    select = selector: makeSet (selector versions);
+    minors = mapAttrs (name: makeSet)
+      (foldl' (acc: el: let minor = lib.substring 0 (lib.stringLength el - 1) el; in 
+        acc // {
+          ${minor} = acc.${minor} or {} // { ${el} = versions.${el}; };
+        }
+      ) {} (lib.attrNames versions));
+  };
 
-  # forGhcs (p: [ p.ghc864 p.ghc863 ])
-  forGhcs = selector: null;
+  result = makeSet (unstable // stable) // {
+    onlyStable = makeSet stable;
+    onlyUnstable = makeSet unstable;
+    allVersions = versions;
+  };
 
-}
+in result

update.hs

@@ -28,17 +28,23 @@ import           System.Process
 import           Text.Regex.Applicative
 
 data Env = Env
-  { gitBin   :: FilePath
-  , nixBin   :: FilePath
-  , cacheDir :: FilePath
-  , manager  :: Manager
+  { gitBin      :: FilePath
+  , nixBin      :: FilePath
+  , nixHashBin  :: FilePath
+  , nixBuildBin :: FilePath
+  , cpBin       :: FilePath
+  , cacheDir    :: FilePath
+  , manager     :: Manager
   }
 
 -- | Initializes the read-only environment
 getEnv :: IO Env
 getEnv = Env
   <$> (fromMaybe (error "Can't find git executable") <$> findExecutable "git")
   <*> (fromMaybe (error "Can't find nix-instantiate executable") <$> findExecutable "nix-instantiate")
+  <*> (fromMaybe (error "Can't find nix-hash executable") <$> findExecutable "nix-hash")
+  <*> (fromMaybe (error "Can't find nix-build executable") <$> findExecutable "nix-build")
+  <*> (fromMaybe (error "Can't find cp executable") <$> findExecutable "cp")
   <*> getXdgDirectory XdgCache "all-hies"
   <*> newTlsManager
 
@@ -64,9 +70,13 @@ run = do
 
 setupDirectories :: App ()
 setupDirectories = do
-  cachePath "per-nixpkgs" >>= liftIO . createDirectoryIfMissing True
+  cachePath "per-nixpkgs/ghcVersions" >>= liftIO . createDirectoryIfMissing True
+  cachePath "per-nixpkgs/sha256" >>= liftIO . createDirectoryIfMissing True
   cachePath "per-hie" >>= liftIO . createDirectoryIfMissing True
+  cachePath "per-ghcMinor" >>= liftIO . createDirectoryIfMissing True
   liftIO $ createDirectoryIfMissing True "nixpkgsForGhc"
+  cleanDirectory "ghcBaseLibraries"
+  cleanDirectory "nixpkgsHashes"
 
 -- | Makes sure that the given directory is existent and empty
 cleanDirectory :: FilePath -> App ()
@@ -122,13 +132,49 @@ genS2N hash version = do
   exists <- liftIO $ doesFileExist path
 
   nixpkgsRev <- findNixpkgsForGhc version
+  sha <- nixpkgsSha nixpkgsRev
+  liftIO $ writeFile ("nixpkgsHashes" </> nixpkgsRev) sha
+
+  genBaseLibraries version nixpkgsRev
 
   unless exists $ do
     liftIO $ putStrLn $ "Using nixpkgs revision " ++ nixpkgsRev ++ " for ghc version " ++ show version
     git nixpkgs [ "checkout", nixpkgsRev ]
     callStack2nix hash path version
   return path
 
+genBaseLibraries :: Version -> String -> App ()
+genBaseLibraries version@(Version major minor patch) nixpkgsRev = do
+  cache <- cachePath $ "per-ghcMinor" </> show major ++ show minor
+  exists <- liftIO $ doesFileExist cache
+  unless exists $ do
+    git nixpkgs [ "checkout", nixpkgsRev ]
+    nix <- lift $ asks nixBuildBin
+    ghcPath <- liftIO $ init <$> readProcess nix
+      [ "--no-out-link", "<nixpkgs>", "-A", "haskell.compiler." ++ nixVersion version ] ""
+    libs <- liftIO $ readProcess (ghcPath </> "bin/ghc-pkg")
+      [ "list", "--no-user-package-db", "--simple" ] ""
+    liftIO $ writeFile cache libs
+  liftIO $ copyFile cache ("ghcBaseLibraries" </> nixVersion version)
+
+nixpkgsSha :: String -> App String
+nixpkgsSha revision = do
+  cacheFile <- cachePath $ "per-nixpkgs/sha256" </> revision
+  exists <- liftIO $ doesFileExist cacheFile
+  if exists then liftIO $ readFile cacheFile
+    else do
+    git nixpkgs [ "checkout", revision ]
+    path <- repoPath nixpkgs
+    tmp <- cachePath "tmp"
+    cp <- lift $ asks cpBin
+    liftIO $ readProcess cp ["-rl", path, tmp] ""
+    liftIO $ removeDirectoryRecursive (tmp </> ".git")
+    nixHash <- lift $ asks nixHashBin
+    hash <- liftIO $ head . lines <$> readProcess nixHash ["--type", "sha256", "--base32", tmp] ""
+    liftIO $ writeFile cacheFile hash
+    liftIO $ removeDirectoryRecursive tmp
+    return hash
+
 -- | Finds a suitable nixpkgs revision that has a the specified compiler available
 findNixpkgsForGhc :: Version -> App String
 findNixpkgsForGhc version = do
@@ -154,7 +200,7 @@ findNixpkgsForGhc version = do
 -- | Determines the available GHC versions for a nixpkgs revision
 ghcVersionsForNixpkgs :: String -> App [Version]
 ghcVersionsForNixpkgs rev = do
-  path <- cachePath $ "per-nixpkgs" </> rev
+  path <- cachePath $ "per-nixpkgs/ghcVersions" </> rev
   exists <- liftIO $ doesFileExist path
   if exists then do
     contents <- liftIO $ readFile path
@@ -253,7 +299,7 @@ getHistory = do
     response <- liftIO $ responseBody <$> httpLbs (parseRequest_ url) mgr
     -- Because this file is downloaded in the order of oldest to newest
     -- We reverse it for easier processing
-    let items = reverse . map (head . BS.words) $ BS.lines response
+    let items = reverse . ("45e819dd49799d8b680084ed57f6d0633581b957":) . map (head . BS.words) $ BS.lines response
 
     liftIO $ BS.writeFile path $ BS.unlines items
     return $ map BS.unpack items