From 6cab8d881f95017dda7bff9760d0df34e689badb Mon Sep 17 00:00:00 2001 From: Leonardo Schwarz Date: Thu, 13 Feb 2025 14:41:39 +0100 Subject: [PATCH] add Makefile for easier app-runner interaction --- bfabric_app_runner/docs/changelog.md | 1 + .../src/bfabric_app_runner/cli/app.py | 20 +++++ .../bfabric_app_runner/resources/workunit.mk | 87 +++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk diff --git a/bfabric_app_runner/docs/changelog.md b/bfabric_app_runner/docs/changelog.md index 87200f1b..46d9661c 100644 --- a/bfabric_app_runner/docs/changelog.md +++ b/bfabric_app_runner/docs/changelog.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added - Implement `--force-storage` to pass a yaml to a forced storage instead of the real one. +- A Makefile will be created in the app folder for easier interaction with the app-runner (it uses uv and PyPI). ### Changed diff --git a/bfabric_app_runner/src/bfabric_app_runner/cli/app.py b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py index 313e6ce0..3064deb2 100644 --- a/bfabric_app_runner/src/bfabric_app_runner/cli/app.py +++ b/bfabric_app_runner/src/bfabric_app_runner/cli/app.py @@ -1,8 +1,10 @@ from __future__ import annotations +import importlib.metadata from pathlib import Path import cyclopts +from loguru import logger from bfabric_app_runner.app_runner.resolve_app import load_workunit_information from bfabric_app_runner.app_runner.runner import run_app, Runner @@ -28,6 +30,8 @@ def run( client = Bfabric.from_config() app_version, workunit_ref = load_workunit_information(app_spec, client, work_dir, workunit_ref) + copy_dev_makefile(work_dir=work_dir) + # TODO(#107): usage of entity lookup cache was problematic -> beyond the full solution we could also consider # to deactivate the cache for the output registration # with EntityLookupCache.enable(): @@ -63,3 +67,19 @@ def dispatch( with EntityLookupCache.enable(): runner = Runner(spec=app_version, client=client, ssh_user=None) runner.run_dispatch(workunit_ref=workunit_ref, work_dir=work_dir) + + +def copy_dev_makefile(work_dir: Path) -> None: + """Copies the workunit.mk file to the work directory, and sets the version of the app runner.""" + source_path = Path(__file__).parents[1] / "resources" / "workunit.mk" + target_path = work_dir / "Makefile" + + makefile_template = target_path.read_text() + app_runner_version = importlib.metadata.version("bfabric_app_runner") + makefile = makefile_template.replace("@RUNNER_VERSION@", app_runner_version) + + if target_path.exists(): + logger.info("Renaming existing Makefile to Makefile.bak") + target_path.rename(work_dir / "Makefile.bak") + logger.info(f"Copying Makefile from {source_path} to {target_path}") + target_path.write_text(makefile) diff --git a/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk b/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk new file mode 100644 index 00000000..559f4476 --- /dev/null +++ b/bfabric_app_runner/src/bfabric_app_runner/resources/workunit.mk @@ -0,0 +1,87 @@ +# Makefile for bfabric-app-runner operations +# +# Quick Start Guide: +# ----------------- +# For most cases, just run: +# make run-all +# +# For step-by-step execution: +# 1. make dispatch # Creates chunks.yml +# 2. make inputs WORK_DIR=dir # Prepares input files +# 3. make process WORK_DIR=dir # Processes the chunks in specified directory +# 4. make stage WORK_DIR=dir # Stages results to server/storage +# +# The WORK_DIR parameter defaults to "work" if not specified. +# Example with custom directory: +# make inputs WORK_DIR=work_folder_1 +# make process WORK_DIR=work_folder_1 +# make stage WORK_DIR=work_folder_1 +# +# Use `make help` to see all available commands + +# Configuration +RUNNER_VERSION := @RUNNER_VERSION@ +RUNNER_CMD := uv run --with bfabric-app-runner==$(RUNNER_VERSION) bfabric-app-runner + +# Input files +APP_DEF := app_definition.yml +WORKUNIT_DEF := $(realpath workunit_definition.yml) + +# Default work directory (can be overridden via command line) +WORK_DIR ?= work + +.PHONY: help dispatch inputs process stage run-all clean + +# Default target +help: + @echo "Available commands:" + @echo " make run-all - Run all steps in a single command (recommended for most cases)" + @echo "" + @echo "Step-by-step execution:" + @echo " make dispatch - Step 1: Initial step (creates chunks.yml)" + @echo " make inputs [WORK_DIR=dir] - Step 2: Prepare input files" + @echo " make process [WORK_DIR=dir] - Step 3: Process chunks in specified directory" + @echo " make stage [WORK_DIR=dir] - Step 4: Stage results to server/storage" + @echo "" + @echo "Other commands:" + @echo " make clean [WORK_DIR=dir] - Remove specified work directory" + @echo " make help - Show this help message" + @echo "" + @echo "Current settings:" + @echo " WORK_DIR = $(WORK_DIR) (default: work)" + +# Step 1: Initial dispatch +dispatch: + @echo "Step 1/4: Running initial dispatch..." + $(RUNNER_CMD) app dispatch $(APP_DEF) $(PWD) $(WORKUNIT_DEF) + @echo "✓ Dispatch completed - chunks.yml created" + +# Step 2: Prepare inputs +inputs: + @echo "Step 2/4: Preparing inputs in directory '$(WORK_DIR)'..." + $(RUNNER_CMD) inputs prepare $(WORK_DIR)/inputs.yml + @echo "✓ Inputs prepared for '$(WORK_DIR)'" + +# Step 3: Process chunks +process: + @echo "Step 3/4: Processing chunks in directory '$(WORK_DIR)'..." + $(RUNNER_CMD) chunk process $(APP_DEF) $(WORK_DIR) + @echo "✓ Processing completed for '$(WORK_DIR)'" + +# Step 4: Stage results +stage: + @echo "Step 4/4: Staging results from directory '$(WORK_DIR)'..." + $(RUNNER_CMD) chunk outputs $(APP_DEF) $(WORK_DIR) $(WORKUNIT_DEF) + @echo "✓ Results staged for '$(WORK_DIR)'" + +# Run all steps in one command +run-all: + @echo "Running all steps in a single command..." + $(RUNNER_CMD) app run $(APP_DEF) . $(WORKUNIT_DEF) + @echo "✓ All steps completed" + +# Clean generated files +clean: + @echo "Cleaning directory '$(WORK_DIR)'..." + rm -rf $(WORK_DIR) + @echo "✓ Clean completed for '$(WORK_DIR)'"