Merge bitcoin/bitcoin#34441: ci: Allow running iwyu CI in worktree

fafdb8f635bc157f55e23890264d12170ecd41ae ci: Allow running iwyu ci in worktree (MarcoFalke)
fab73e213dee1057e9e759133767b17ec5b1f6ab ci: Reject unsafe execution of shell scripts (MarcoFalke)

Pull request description:

  Currently, the iwyu CI fails to run in a git-worktree, or git-archive. This is due to the use of `git diff`.

  Fix this by force-initializing a dummy git repo with a single dummy commit.

  It may be possible to detect when `git diff` is not available in the directory, and only apply the fallback when needed, but the git history is not needed and it is easier to unconditionally apply the git init.

ACKs for top commit:
  willcl-ark:
    reACK fafdb8f635bc157f55e23890264d12170ecd41ae
  hebasto:
    ACK fafdb8f635bc157f55e23890264d12170ecd41ae, I have reviewed the code and it looks OK. Tested on Fedora 43.
  sedited:
    ACK fafdb8f635bc157f55e23890264d12170ecd41ae

Tree-SHA512: 572f1e2b9e215c2804095382498abb5b8636e3a49d5ba2a736b975e06afa2881d815b854a8a593d0f187c7c6b55034688e11f46d6814edfe7c29505197e80b18

ci/test/01_base_install.sh

@@ -8,6 +8,11 @@ export LC_ALL=C.UTF-8
 
 set -o errexit -o pipefail -o xtrace
 
+if [ "${DANGER_RUN_CI_ON_HOST}" != "1" ]; then
+  echo "This script will make unsafe local and global modifications, so it can only be run inside a container and requires DANGER_RUN_CI_ON_HOST=1"
+  exit 1
+fi
+
 CFG_DONE="${BASE_ROOT_DIR}/ci.base-install-done"  # Use a global setting to remember whether this script ran to avoid running it twice
 
 if [ "$( cat "${CFG_DONE}" || true )" == "done" ]; then

ci/test/02_run_container.py

@@ -158,7 +158,13 @@ def main():
         if os.getenv("DANGER_RUN_CI_ON_HOST"):
             prefix = []
         else:
-            prefix = ["docker", "exec", container_id]
+            prefix = [
+                "docker",
+                "exec",
+                "--env",
+                "DANGER_RUN_CI_ON_HOST=1",  # Safe to set *inside* the container
+                container_id,
+            ]
 
         return run([*prefix, *cmd_inner], **kwargs)
 

ci/test/03_test_script.sh

@@ -6,7 +6,12 @@
 
 export LC_ALL=C.UTF-8
 
-set -ex
+set -o errexit -o xtrace
+
+if [ "${DANGER_RUN_CI_ON_HOST}" != "1" ]; then
+  echo "This script will make unsafe local and global modifications, so it can only be run inside a container and requires DANGER_RUN_CI_ON_HOST=1"
+  exit 1
+fi
 
 cd "${BASE_ROOT_DIR}"
 
@@ -44,6 +49,19 @@ echo "=== BEGIN env ==="
 env
 echo "=== END env ==="
 
+# The CI framework should be flexible where it is run from. For example, from
+# a git-archive, a git-worktree, or a normal git repo.
+# The iwyu task requires a working git repo, which may not always be
+# available, so initialize one with force.
+if [[ "${RUN_IWYU}" == true ]]; then
+  mv .git .git_ci_backup || true
+  git init
+  git add ./src  # the git diff command used later for iwyu only cares about ./src
+  git config user.email "ci@ci"
+  git config user.name "CI"
+  git commit -m "dummy CI ./src init for IWYU"
+fi
+
 if [ "$RUN_FUZZ_TESTS" = "true" ]; then
   export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_corpora/
   if [ ! -d "$DIR_FUZZ_IN" ]; then

ci/test_imagefile

@@ -21,4 +21,4 @@ COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh ./ci/t
 # Bash is required, so install it when missing
 RUN sh -c "bash -c 'true' || ( apk update && apk add --no-cache bash )"
 
-RUN ["bash", "-c", "cd /ci_container_base/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"]
+RUN ["bash", "-c", "cd /ci_container_base/ && set -o errexit && source ./ci/test/00_setup_env.sh && DANGER_RUN_CI_ON_HOST=1 ./ci/test/01_base_install.sh"]