fetchTree: Allow fetching plain files

Replace the `tarball` fetcher by a more generic `url` one, mostly
similar except that it takes an extra `unpack` parameter (false by
default) to specify whether the target is an archive to extract or a
plain file.

Also adds a few convolutions for backwards compatibility:
- The `tarball` fetcher type is kept. It’s now an alias for `url`, but
  with `unpack` defaulting to `true`
- Flake references that used to resolve to a `tarball` fetcher (i.e.
  `http(s)://` and `file://` urls whose path ends with a known archive
  extension) still do, meaning that
  1. They are still extracted by default (which is both a
     backwards-compatibility measure, and very handy)
  2. The generated lockfiles are still compatible with older Nix
     versions who don’t know of the `url` fetcher

Fix #3785

Co-Authored-By: Tony Olagbaiye <[email protected]>

Author: bqv

doc/manual/src/release-notes/rl-next.md

@@ -24,3 +24,6 @@
 
   Selecting derivation outputs using the attribute selection syntax
   (e.g. `nixpkgs#glibc.dev`) no longer works.
+
+* `builtins.fetchTree` (and flake inputs) can now be used to fetch plain files
+  over the `http(s)` and `file` protocols in addition to directory tarballs.

src/libfetchers/tarball.cc

@@ -168,23 +168,37 @@ std::pair<Tree, time_t> downloadTarball(
     };
 }
 
-struct TarballInputScheme : InputScheme
+struct CurlInputScheme : InputScheme
 {
+    const std::string inputType = "url";
+
+    // For backwards-compatibility.
+    // An alias for `url`, except that setting type to `tarball` will cause
+    // `unpack` to default to `true`
+    const std::string tarballInputType = "tarball";
+
+    const std::string unpackParam = "unpack";
+
     std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
         if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {};
 
-        if (!hasSuffix(url.path, ".zip")
-            && !hasSuffix(url.path, ".tar")
-            && !hasSuffix(url.path, ".tgz")
-            && !hasSuffix(url.path, ".tar.gz")
-            && !hasSuffix(url.path, ".tar.xz")
-            && !hasSuffix(url.path, ".tar.bz2")
-            && !hasSuffix(url.path, ".tar.zst"))
-            return {};
-
         Input input;
-        input.attrs.insert_or_assign("type", "tarball");
+
+        bool defaultUnpack = (hasSuffix(url.path, ".zip")
+            || hasSuffix(url.path, ".tar")
+            || hasSuffix(url.path, ".tgz")
+            || hasSuffix(url.path, ".tar.gz")
+            || hasSuffix(url.path, ".tar.xz")
+            || hasSuffix(url.path, ".tar.bz2")
+            || hasSuffix(url.path, ".tar.zst"));
+
+        if (auto unpack = url.query.find(unpackParam); unpack != url.query.end())
+            input.attrs.insert_or_assign(unpackParam, Explicit<bool> { unpack->second == "1" });
+
+        // To keep things backwards-compatible use the "tarball" input type for
+        // everything that used to be a "tarball" before.
+        input.attrs.insert_or_assign("type", defaultUnpack ? tarballInputType : inputType);
         input.attrs.insert_or_assign("url", url.to_string());
         auto narHash = url.query.find("narHash");
         if (narHash != url.query.end())
@@ -194,14 +208,17 @@ struct TarballInputScheme : InputScheme
 
     std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
-        if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
+        auto type = maybeGetStrAttr(attrs, "type");
+        if (type != inputType && type != tarballInputType) return {};
 
+        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack"};
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name")
-                throw Error("unsupported tarball input attribute '%s'", name);
+            if (!allowedNames.count(name))
+                throw Error("unsupported %s input attribute '%s'", *type, name);
 
         Input input;
         input.attrs = attrs;
+
         //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash");
         return input;
     }
@@ -213,6 +230,8 @@ struct TarballInputScheme : InputScheme
         // don't have a canonical representation.
         if (auto narHash = input.getNarHash())
             url.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
+        if (auto unpack = maybeGetBoolAttr(input.attrs, unpackParam))
+            url.query.insert_or_assign(unpackParam, unpack.value() ? "1" : "0");
         /*
         else if (auto hash = maybeGetStrAttr(input.attrs, "hash"))
             url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true));
@@ -225,13 +244,32 @@ struct TarballInputScheme : InputScheme
         return true;
     }
 
+    /**
+     * Whether this url input should be unpacked when we fetch it
+     */
+    const bool shouldUnpack(const Input & input) const
+    {
+        auto type = getStrAttr(input.attrs, "type");
+        // For backwards-compatibility, we unpack by default if the type is
+        // set to "tarball"
+        return maybeGetBoolAttr(input.attrs, unpackParam).value_or(type == "tarball");
+    }
+
     std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
-        return {std::move(tree.storePath), input};
+        StorePath outPath = [&]() {
+            if (shouldUnpack(input)) {
+                auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
+                return tree.storePath;
+            } else {
+                auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false);
+                return file.storePath;
+            }
+        }();
+        return {std::move(outPath), input};
     }
 };
 
-static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
+static auto rCurlInputScheme = OnStartup([] { registerInputScheme(std::make_unique<CurlInputScheme>()); });
 
 }

src/nix/flake.md

@@ -178,12 +178,20 @@ Currently the `type` attribute can be one of the following:
   `git` type, except that the URL schema must be one of `hg+http`,
   `hg+https`, `hg+ssh` or `hg+file`.
 
-* `tarball`: Tarballs. The location of the tarball is specified by the
-  attribute `url`.
+* `url`: Plain files or directory tarballs, either over http(s) or from the local
+  disk.
 
-  In URL form, the schema must be `http://`, `https://` or `file://`
-  URLs and the extension must be `.zip`, `.tar`, `.tgz`, `.tar.gz`,
-  `.tar.xz`, `.tar.bz2` or `.tar.zst`.
+  In URL form, the schema must be `http://`, `https://` or `file://` URLs.
+  If the extension corresponds to a known archive format (`.zip`, `.tar`,
+  `.tgz`, `.tar.gz`, `.tar.xz`, `.tar.bz2` or `.tar.zst`), the target will be
+  interpreted as a directory tarball to unpack, otherwise it will be treated as
+  a plain file.
+
+  The `unpack` parameter can be passed (both to the URL and the record form) to
+  explicitely specify what to do with the target.
+
+* `tarball`: Tarballs. An alias to `url`, but that will unpack its target as a
+  directory tarball by default.
 
 * `github`: A more efficient way to fetch repositories from
   GitHub. The following attributes are required:
@@ -393,7 +401,7 @@ inputs.nixpkgs = {
 };
 ```
 
-Repositories that don't contain a `flake.nix` can also be used as
+Repositories that don't contain a `flake.nix` or plain files can also be used as
 inputs, by setting the input's `flake` attribute to `false`:
 
 ```nix

tests/fetchTree-file.sh

@@ -0,0 +1,107 @@
+source common.sh
+
+clearStore
+
+cd "$TEST_ROOT"
+
+test_fetch_file () {
+    echo foo > test_input
+
+    input_hash="$(nix hash path test_input)"
+
+    nix eval --impure --file - <<EOF
+    let
+        tree1 = builtins.fetchTree { type = "url"; url = "file://$PWD/test_input"; };
+        tree2 = builtins.fetchTree { type = "tarball"; url = "file://$PWD/test_input"; unpack = false; };
+    in
+    assert (tree1.narHash == "$input_hash");
+    assert (tree2.narHash == "$input_hash");
+    tree1
+EOF
+}
+
+# Make sure that `http(s)` and `file` flake inputs are properly extracted when
+# they should be, and treated as opaque files when they should be
+test_file_flake_input () {
+    rm -fr "$TEST_ROOT/testFlake";
+    mkdir "$TEST_ROOT/testFlake";
+    pushd testFlake
+
+    mkdir inputs
+    echo foo > inputs/test_input_file
+    tar cfa test_input.tar.gz inputs
+    cp test_input.tar.gz test_input_no_ext
+    input_tarball_hash="$(nix hash path test_input.tar.gz)"
+    input_directory_hash="$(nix hash path inputs)"
+
+    cat <<EOF > flake.nix
+    {
+        inputs.no_ext_default_no_unpack = {
+            url = "file://$PWD/test_input_no_ext";
+            flake = false;
+        };
+        inputs.no_ext_explicit_unpack = {
+            url = "file://$PWD/test_input_no_ext?unpack=1";
+            flake = false;
+        };
+        inputs.tarball_default_unpack = {
+            url = "file://$PWD/test_input.tar.gz";
+            flake = false;
+        };
+        inputs.tarball_explicit_no_unpack = {
+            url = "file://$PWD/test_input.tar.gz?unpack=0";
+            flake = false;
+        };
+        outputs = { ... }: {};
+    }
+EOF
+
+    nix flake update
+    nix eval --file - <<EOF
+    with (builtins.fromJSON (builtins.readFile ./flake.lock));
+
+    # Url inputs whose extension doesn’t match a know archive format should
+    # not be unpacked by default
+    assert (nodes.no_ext_default_no_unpack.locked.type == "url");
+    assert (nodes.no_ext_default_no_unpack.locked.unpack or false == false);
+    assert (nodes.no_ext_default_no_unpack.locked.narHash == "$input_tarball_hash");
+
+    # For backwards compatibility, flake inputs that correspond to the
+    # old 'tarball' fetcher should still have their type set to 'tarball'
+    assert (nodes.tarball_default_unpack.locked.type == "tarball");
+    # Unless explicitely specified, the 'unpack' parameter shouldn’t appear here
+    # because that would break older Nix versions
+    assert (!nodes.tarball_default_unpack.locked ? unpack);
+    assert (nodes.tarball_default_unpack.locked.narHash == "$input_directory_hash");
+
+    # Explicitely passing the unpack parameter should enforce the desired behavior
+    assert (nodes.no_ext_explicit_unpack.locked.narHash == nodes.tarball_default_unpack.locked.narHash);
+    assert (nodes.tarball_explicit_no_unpack.locked.narHash == nodes.no_ext_default_no_unpack.locked.narHash);
+    true
+EOF
+    popd
+
+    [[ -z "${NIX_DAEMON_PACKAGE}" ]] && return 0
+
+    # Ensure that a lockfile generated by the current Nix for tarball inputs
+    # can still be read by an older Nix
+
+    cat <<EOF > flake.nix
+    {
+        inputs.tarball = {
+            url = "file://$PWD/test_input.tar.gz";
+            flake = false;
+        };
+        outputs = { self, tarball }: {
+            foo = builtins.readFile "${tarball}/test_input_file";
+        };
+    }
+    nix flake update
+
+    clearStore
+    "$NIX_DAEMON_PACKAGE/bin/nix" eval .#foo
+EOF
+}
+
+test_fetch_file
+test_file_flake_input

tests/local.mk

@@ -23,6 +23,7 @@ nix_tests = \
   fetchGit.sh \
   fetchurl.sh \
   fetchPath.sh \
+  fetchTree-file.sh \
   simple.sh \
   referrers.sh \
   optimise-store.sh \

tests/tarball.sh

@@ -31,6 +31,7 @@ test_tarball() {
 
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
+    nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"url\"; url = file://$tarball; unpack = true; })"
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
     # Do not re-fetch paths already present
     nix-build  -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })"