diff --git a/cmd/earthly/subcmd/build_cmd.go b/cmd/earthly/subcmd/build_cmd.go
index 022aeb63..9c741579 100644
--- a/cmd/earthly/subcmd/build_cmd.go
+++ b/cmd/earthly/subcmd/build_cmd.go
@@ -852,8 +852,7 @@ func (a *Build) initAutoSkip(ctx context.Context, target domain.Target, overridi
 		EarthlyCIRunner: a.cli.Flags().EarthlyCIRunner,
 	})
 	if err != nil {
-		console.Warnf("unable to calculate hash for %s: %s", target.String(), err.Error())
-		return nil, nil, false, nil
+		return nil, nil, false, errors.Wrapf(err, "unable to calculate hash for %s", target)
 	}
 
 	skipDB, err = bk.NewBuildkitSkipper(a.cli.Flags().LocalSkipDB, orgName, projectName, target.GetName(), client)
diff --git a/tests/autoskip/Earthfile b/tests/autoskip/Earthfile
index 6a969c7c..96cb8d51 100644
--- a/tests/autoskip/Earthfile
+++ b/tests/autoskip/Earthfile
@@ -54,8 +54,7 @@ test-with-subdir:
 
 test-requires-project:
     RUN echo hello > my-file
-    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=no-project.earth --target=+no-project --output_contains="I was run"
-    RUN if ! grep "PROJECT command missing" earthly.output >/dev/null; then echo "no warning displayed for missing PROJECT keyword" && exit 1; fi
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=no-project.earth --target=+no-project --should_fail=true --output_contains="command missing"
 
 test-wait:
     DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="not skipped"
@@ -161,16 +160,12 @@ test-shell-out:
     DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out
     DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out --output_contains="target .* has already been run; exiting"
 
-    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out-target --output_contains="dynamic target"
-    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out-target --output_does_not_contain="target .* has already been run; exiting"
-
-    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out-target-2 --output_contains="dynamic target"
-    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out-target-2 --output_does_not_contain="target .* has already been run; exiting"
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out-target --should_fail=true --output_contains="dynamic target"
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out-target-2 --should_fail=true --output_contains="dynamic target"
 
     RUN echo "foo" > my-file
 
-    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out-copy --output_contains="dynamic COPY source"
-    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out-copy --output_does_not_contain="target .* has already been run; exiting"
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=shell-out.earth --target=+shell-out-copy --should_fail=true --output_contains="dynamic COPY source"
 
 test-copy-if-exists:
     DO --pass-args +RUN_EARTHLY_ARGS --earthfile=simple.earth --target=+copy-if-exists
@@ -193,12 +188,12 @@ test-remote-targets:
     DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-from-sha
     DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-from-sha --output_contains="target .* has already been run; exiting"
 
-    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+invalid-copy-branch --output_contains="complete Git SHA or tag"
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+invalid-copy-branch --should_fail=true --output_contains="complete Git SHA or tag"
 
     DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-build
     DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-build --output_contains="target .* has already been run; exiting"
 
-    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+invalid-build --output_contains="complete Git SHA or tag"
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+invalid-build --should_fail=true --output_contains="complete Git SHA or tag"
 
 test-from-dockerfile:
     RUN mkdir /dist