From 073b77f16b1409cf9f855a1236305dfe2d3b4409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norman=20B=C3=B6wing?= Date: Tue, 5 May 2026 15:11:36 +0200 Subject: [PATCH 1/3] Add log helper script --- serverless-fleets/README.md | 24 ++++++++++---- serverless-fleets/fleet_logs | 63 ++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 7 deletions(-) create mode 100755 serverless-fleets/fleet_logs diff --git a/serverless-fleets/README.md b/serverless-fleets/README.md index 6b869776b..6788a8c26 100644 --- a/serverless-fleets/README.md +++ b/serverless-fleets/README.md @@ -46,7 +46,7 @@ A fleet consists of a collection of worker nodes that automatically scale up or Like applications, jobs, and functions, fleets run within a Code Engine project. A project is a grouping of Code Engine resources within a specific IBM Cloud region. Projects are used to organize resources and manage access to entities such as configmaps, secrets, and persistent data stores. -## Architecture +## Architecture The architecture used in this tutorial looks as follows. @@ -75,7 +75,7 @@ The tutorial has been tested on a MacOS and Ubuntu24 client machine with the fol Clone this repository ``` -git clone https://github.com/IBM/CodeEngine.git +git clone https://github.com/IBM/CodeEngine.git ``` Switch to the `serverless-fleets` directory, which will be the root directory for all steps of this tutorial @@ -295,7 +295,7 @@ Run a serverless fleet to process 100 tasks where each tasks gets 1 CPU and 2 GB
-In the fleet details you will see 5 workers being provisined. The number of workers is determined by the profile, cpu/memory and number of parallel tasks. +In the fleet details you will see 5 workers being provisined. The number of workers is determined by the profile, cpu/memory and number of parallel tasks. ``` ibmcloud ce fleet get --id @@ -413,7 +413,7 @@ The 6 tasks are submitted using the `tasks-from-local-file` option using the [wo ![](./images/example_wordcount.png) -The example mounts the [Persistant Data Stores](https://cloud.ibm.com/docs/codeengine?topic=codeengine-persistent-data-store) (PDS) to the container using the `--mount-data-store MOUNT_DIRECTORY=STORAGE_NAME:[SUBPATH]`, where +The example mounts the [Persistant Data Stores](https://cloud.ibm.com/docs/codeengine?topic=codeengine-persistent-data-store) (PDS) to the container using the `--mount-data-store MOUNT_DIRECTORY=STORAGE_NAME:[SUBPATH]`, where - `MOUNT_DIRECTORY` - is the directory within the container - `STORAGE_NAME` - is the name of the PDS - `SUBPATH` - is the prefix within the COS bucket to mount. @@ -434,10 +434,10 @@ Upload the .txt files from the local data directory to Cloud Object Storage #### Step 2 - Run the fleet -Launch the fleet to perform `wc` on each of the novels which defines the tasks from [wordcount_commands.jsonl](./wordcount_commands.jsonl) and mounts the input and output data stores. +Launch the fleet to perform `wc` on each of the novels which defines the tasks from [wordcount_commands.jsonl](./wordcount_commands.jsonl) and mounts the input and output data stores. ``` ./run_wordcount -``` +``` Confirm that you uploaded the files with `#? 1` @@ -506,7 +506,7 @@ Download the results from the output COS bucket to `./data/output` ``` ./download -```` +```` 🚀 The example was successful, if you can tell the number of words of the "Alice in Wonderland" novel 🚀 @@ -555,6 +555,16 @@ An IBM Cloud Logs instance is being setup and enabled by default during the auto ![](./images/prototype_logs.png) +![](./images/prototype_logs.png) + +#### Using helper scripts + +If you want to quickly iterate you may use the helper script `fleet-logs` in order to display fleet logs directly in your terminal. The scripts uses a `LOGS_URL` env variable targetting your ICL endpoint, or if unset will try to find that ICL endpoint of your ICL instance. Additionally, you may specify `--fleet-id `, `--since `, `--tier ` and `--output ` to parameterize your query. Available values are directly taken from `ibmcloud logs query --help`. + +``` +./fleet-logs --fleet-id +``` + ### Cleanup the Environment To clean up all IBM Cloud resources, that have been created as part of the provided script, run: diff --git a/serverless-fleets/fleet_logs b/serverless-fleets/fleet_logs new file mode 100755 index 000000000..3bc6ad9d9 --- /dev/null +++ b/serverless-fleets/fleet_logs @@ -0,0 +1,63 @@ +#!/bin/bash +set -eo pipefail + +ICL_ENDPOINT="${LOGS_URL}" +REGION="${REGION:=eu-de}" +NAME_PREFIX="${NAME_PREFIX:=ce-fleet-sandbox}" +RESOURCE_GROUP="${NAME_PREFIX}--rg" +ICL_NAME="${NAME_PREFIX}--icl" +FLEET_ID= +TIER="frequent_search" +OUTPUT="logs-raw" +SINCE="6h0m0s" + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source ${SCRIPT_DIR}/common.sh + +target_region $REGION > /dev/null +target_resource_group $RESOURCE_GROUP > /dev/null + +ibmcloud plugin show logs > /dev/null + +if [[ $? -ne 0 ]]; then + ibmcloud plugin install logs +fi + +if [[ -z "${ICL_ENDPOINT}" ]]; then + ICL_ENDPOINT="$(ibmcloud resource service-instance ${ICL_NAME} -o json | jq -r '.[].extensions.external_api')" +fi + +while [[ $# -gt 0 ]]; do + case "$1" in + --fleet-id) + shift + FLEET_ID="$1" + ;; + + --tier) + shift + TIER="$1" + ;; + + --output) + shift + OUTPUT="$1" + ;; + + --since) + shift + SINCE="$1" + ;; + + *) + echo "Unknown flag $1" + exit 1 + ;; + esac + shift +done +if [ -z "$FLEET_ID" ]; then + ibmcloud logs query --since ${SINCE} --tier ${TIER} --query "source logs | sort by \$m.timestamp" --service-url "${ICL_ENDPOINT}" --output ${OUTPUT} +else + ibmcloud logs query --since ${SINCE} --tier ${TIER} --query "filter \$d.codeengine.fleetId ~ '${FLEET_ID}' | sort by \$m.timestamp" --service-url "${ICL_ENDPOINT}" --output ${OUTPUT} +fi \ No newline at end of file From 53f58f6fc829eb4f18c4817b0f4890dd4f978a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norman=20B=C3=B6wing?= Date: Wed, 27 May 2026 07:53:19 +0200 Subject: [PATCH 2/3] Add more options to fleet-logs helper script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Norman Böwing --- serverless-fleets/README.md | 4 +- serverless-fleets/fleet_logs | 73 +++++++++++++++++++++++++----------- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/serverless-fleets/README.md b/serverless-fleets/README.md index 72dcf3050..de843ac3c 100644 --- a/serverless-fleets/README.md +++ b/serverless-fleets/README.md @@ -562,10 +562,10 @@ An IBM Cloud Logs instance is being setup and enabled by default during the auto #### Using helper scripts -If you want to quickly iterate you may use the helper script `fleet-logs` in order to display fleet logs directly in your terminal. The scripts uses a `LOGS_URL` env variable targetting your ICL endpoint, or if unset will try to find that ICL endpoint of your ICL instance. Additionally, you may specify `--fleet-id `, `--since `, `--tier ` and `--output ` to parameterize your query. Available values are directly taken from `ibmcloud logs query --help`. +If you want to quickly iterate you may use the helper script `fleet-logs` in order to display fleet logs directly in your terminal. The scripts uses a `LOGS_URL` env variable targetting your ICL endpoint, or if unset will try to find that ICL endpoint of your ICL instance. Additionally, you may specify `--fleet-id `, `--task-id `, `--subcomponent-type [fleet_worker|fleet_task]`, `--since `, `--tier ` and `--output ` to parameterize your query. Available values are directly taken from `ibmcloud logs query --help`. ``` -./fleet-logs --fleet-id +./fleet_logs --tier --fleet-id --task-id --since 6h0m0s ``` ### How to customize fleet workers diff --git a/serverless-fleets/fleet_logs b/serverless-fleets/fleet_logs index 3bc6ad9d9..f9767d75c 100755 --- a/serverless-fleets/fleet_logs +++ b/serverless-fleets/fleet_logs @@ -1,31 +1,24 @@ #!/bin/bash set -eo pipefail +usage() { + echo "Example usage: ./fleet_logs --tier --fleet-id --task-id --since 6h0m0s" + exit 1 +} + ICL_ENDPOINT="${LOGS_URL}" REGION="${REGION:=eu-de}" NAME_PREFIX="${NAME_PREFIX:=ce-fleet-sandbox}" -RESOURCE_GROUP="${NAME_PREFIX}--rg" +RESOURCE_GROUP="${RESOURCE_GROUP:=$NAME_PREFIX--rg}" ICL_NAME="${NAME_PREFIX}--icl" FLEET_ID= +TASK_ID= +SUBCOMPONENT_TYPE= TIER="frequent_search" OUTPUT="logs-raw" SINCE="6h0m0s" SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source ${SCRIPT_DIR}/common.sh - -target_region $REGION > /dev/null -target_resource_group $RESOURCE_GROUP > /dev/null - -ibmcloud plugin show logs > /dev/null - -if [[ $? -ne 0 ]]; then - ibmcloud plugin install logs -fi - -if [[ -z "${ICL_ENDPOINT}" ]]; then - ICL_ENDPOINT="$(ibmcloud resource service-instance ${ICL_NAME} -o json | jq -r '.[].extensions.external_api')" -fi while [[ $# -gt 0 ]]; do case "$1" in @@ -34,6 +27,16 @@ while [[ $# -gt 0 ]]; do FLEET_ID="$1" ;; + --task-id) + shift + TASK_ID="$1" + ;; + + --subcomponent-type) + shift + SUBCOMPONENT_TYPE="$1" + ;; + --tier) shift TIER="$1" @@ -49,15 +52,41 @@ while [[ $# -gt 0 ]]; do SINCE="$1" ;; + --help) + usage + ;; + *) - echo "Unknown flag $1" - exit 1 + usage ;; esac shift done -if [ -z "$FLEET_ID" ]; then - ibmcloud logs query --since ${SINCE} --tier ${TIER} --query "source logs | sort by \$m.timestamp" --service-url "${ICL_ENDPOINT}" --output ${OUTPUT} -else - ibmcloud logs query --since ${SINCE} --tier ${TIER} --query "filter \$d.codeengine.fleetId ~ '${FLEET_ID}' | sort by \$m.timestamp" --service-url "${ICL_ENDPOINT}" --output ${OUTPUT} -fi \ No newline at end of file + +source ${SCRIPT_DIR}/common.sh + +target_region $REGION > /dev/null +target_resource_group $RESOURCE_GROUP > /dev/null + +ibmcloud plugin show logs > /dev/null + +if [[ $? -ne 0 ]]; then + ibmcloud plugin install logs +fi + +if [[ -z "${ICL_ENDPOINT}" ]]; then + ICL_ENDPOINT=https://"$(ibmcloud resource service-instance ${ICL_NAME} -o json | jq -r '.[].extensions.external_api')" +fi + +LOGS_FILTER="source logs" +if [ -n "$FLEET_ID" ]; then + LOGS_FILTER="${LOGS_FILTER} | filter \$d.codeengine.fleetId ~ '${FLEET_ID}'" +fi +if [ -n "$TASK_ID" ]; then + LOGS_FILTER="${LOGS_FILTER} | filter \$d.codeengine.taskId ~ '${TASK_ID}'" +fi +if [ -n "$SUBCOMPONENT_TYPE" ]; then + LOGS_FILTER="${LOGS_FILTER} | filter \$d.codeengine.subcomponentType ~ '${SUBCOMPONENT_TYPE}'" +fi + +ibmcloud logs query --since ${SINCE} --tier ${TIER} --query "${LOGS_FILTER} | sort by \$m.timestamp" --service-url "${ICL_ENDPOINT}" --output ${OUTPUT} From 68c42471f7b11fc36e3c9a227ae07baecaa838ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norman=20B=C3=B6wing?= Date: Wed, 27 May 2026 09:01:16 +0200 Subject: [PATCH 3/3] Improve examples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Norman Böwing --- serverless-fleets/README.md | 2 +- serverless-fleets/fleet_logs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/serverless-fleets/README.md b/serverless-fleets/README.md index de843ac3c..9d6a358a7 100644 --- a/serverless-fleets/README.md +++ b/serverless-fleets/README.md @@ -565,7 +565,7 @@ An IBM Cloud Logs instance is being setup and enabled by default during the auto If you want to quickly iterate you may use the helper script `fleet-logs` in order to display fleet logs directly in your terminal. The scripts uses a `LOGS_URL` env variable targetting your ICL endpoint, or if unset will try to find that ICL endpoint of your ICL instance. Additionally, you may specify `--fleet-id `, `--task-id `, `--subcomponent-type [fleet_worker|fleet_task]`, `--since `, `--tier ` and `--output ` to parameterize your query. Available values are directly taken from `ibmcloud logs query --help`. ``` -./fleet_logs --tier --fleet-id --task-id --since 6h0m0s +./fleet_logs --tier --fleet-id --task-id --subcomponent-type [fleet_task|fleet_worker] --since 6h0m0s ``` ### How to customize fleet workers diff --git a/serverless-fleets/fleet_logs b/serverless-fleets/fleet_logs index f9767d75c..eaea648c0 100755 --- a/serverless-fleets/fleet_logs +++ b/serverless-fleets/fleet_logs @@ -2,7 +2,7 @@ set -eo pipefail usage() { - echo "Example usage: ./fleet_logs --tier --fleet-id --task-id --since 6h0m0s" + echo "Example usage: ./fleet_logs --tier --fleet-id --task-id --subcomponent-type [fleet_task|fleet_worker] --since 6h0m0s" exit 1 }