Merge pull request #28 from thin-edge/chore-streamline-project

chore: project restructuring

.gitignore

@@ -1,7 +1,3 @@
-/*/
-!/projects/
-!/projects/*
-!/conf/
-!/scripts/
-!.github/
+_kas/
+build/
 .env

README.md

@@ -45,7 +45,7 @@ It is highly recommended to use **Ubuntu 20.04 LTS** for your build (host) machi
 #### Minimum requirements
 
 * At least 100 GB free disk space (however ~500GB is recommended as you don't want to run out and have to set all of this up again)
-* x86_64 Machine if you have it, however arm64/aarch64 can also work 
+* x86_64 Machine (arm64/aarch64 is not supported by Yocto)
 * Lots of time (first build can take > 6 hours...welcome to the world of yocto)
 
 
@@ -67,21 +67,12 @@ To start off, you will need to install the project and Yocto dependencies.
 
     **Ubuntu 20.04 LTS**
 
-    ```
+    ```sh
     sudo apt install file gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev xterm python3-subunit mesa-common-dev zstd liblz4-tool
     ```
 
     If you having troubles please consult the [Yocto Documentation](https://docs.yoctoproject.org/kirkstone/brief-yoctoprojectqs/index.html#building-your-image)
 
-5. Install git-lfs dependency (this will be removed in the future)
-
-    ```sh
-    curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
-    sudo apt-get install git-lfs
-    ```
-
-    If you have any problems with the installation of git-lfs, please check the [official git-lfs instructions for linux](https://github.com/git-lfs/git-lfs/blob/main/INSTALLING.md).
-
 **Reducing build times**
 
 You can configure a common **sstate cache** and download folder to share amongst different projects to speed up the build times.
@@ -97,13 +88,13 @@ DL_DIR=/data/yocto/downloads
 
 ### Building an image
 
-Build the demo image (which includes thin-edge.io and mender standalone)
+Build an image which includes thin-edge.io and [RAUC](https://rauc.readthedocs.io/en/latest/):
 
 **Raspberry Pi**
 
 ```sh
-KAS_MACHINE=raspberrypi3-64 just build-demo
-KAS_MACHINE=raspberrypi4-64 just build-demo
+KAS_MACHINE=raspberrypi3-64 just build-project ./projects/tedge-rauc.yaml
+KAS_MACHINE=raspberrypi4-64 just build-project ./projects/tedge-rauc.yaml
 ```
 
 Or you can save the KAS_MACHINE value in your .env file
@@ -114,8 +105,8 @@ KAS_MACHINE=raspberrypi3-64
 
 The bootstrap image (which you can flash to an SD card) is located below:
 
-```
-ls -ltr build/tmp/deploy/images/$KAS_MACHINE/*.sdimg.bz2
+```sh
+ls -ltr "build/tmp/deploy/images/$KAS_MACHINE"/*.bz2
 ```
 
 Where KAS_MACHINE should be replaced with the target machine value, e.g. `raspberrypi4-64`.
@@ -137,7 +128,7 @@ just publish
 The projects use lock files in order to create reproducible builds by fixing layers to specific commits. Therefore if you need to update them, then run the following task:
 
 ```sh
-just update-demo-lock
+just update-all-locks
 ```
 
 Or you can specify which project file should be updated instead (if you want more control over it).
@@ -155,3 +146,9 @@ If you run into unexpected build errors, or the image just doesn't include the e
 ```sh
 just clean
 ```
+
+Or if you are still having problems, try removing all of the temp folders (build and the repositories)
+
+```sh
+just clean-all
+```

justfile

@@ -1,9 +1,19 @@
 set dotenv-load
 
+# Use custom work_dir for kas, so the folder can be added to the gitignore list
+# but keep the build dir under <project>/build
+export KAS_WORK_DIR := env_var_or_default("KAS_WORK_DIR", justfile_directory() + "/_kas")
+export KAS_BUILD_DIR := env_var_or_default("KAS_BUILD_DIR", justfile_directory() + "/build")
+
 # Use an auto generated image version suffix (based on date) if one is not provided
 export IMAGE_VERSION := env_var_or_default("IMAGE_VERSION", "" + `date +%Y%m%d.%H%M`)
 export IMAGE_VERSION_SUFFIX := "_" + IMAGE_VERSION
 
+# Init build folders
+init:
+    mkdir -p "{{KAS_WORK_DIR}}"
+    mkdir -p "{{KAS_BUILD_DIR}}"
+
 # Generate a build version
 generate-version:
     @echo "{{IMAGE_VERSION_SUFFIX}}"
@@ -13,28 +23,26 @@ clean:
     @echo "Cleaning tmp folder"
     rm -rf build/tmp
 
+# Clean all build and yocto folders
+clean-all:
+    @echo "Cleaning all local yocto folders"
+    rm -rf build
+    rm -rf _kas
+
 # Update project lock file to lock to specific commit per layer
 # You need to run this if you are working on a branch and would like up-to-date
 # changes
-update-lock file *args="":
+update-lock file *args="": init
     python3 -m kas dump --update --lock {{args}} {{file}} --inplace
 
-# Update the lock file used by the main demo (convinience task)
-update-demo-lock *args="":
-    just update-lock ./projects/tedge-rauc.yaml {{args}}
-
 # Build project from a given file
-build-project file *args="":
+build-project file *args="": init
     python3 -m kas build {{file}} {{args}}
 
 # Run custom bitbake command with a given project
-bitbake file *args="":
+bitbake file *args="": init
     python3 -m kas shell {{file}} -c 'bitbake {{args}}'
 
-# Build demo
-build-demo *args="":
-    python3 -m kas build ./projects/tedge-rauc.yaml {{args}}
-
 # Publish image to Cumulocity IoT (requires go-c8y-cli to be installed)
 publish *args="":
     {{justfile_directory()}}/scripts/publish-c8y.sh {{args}}

scripts/publish-c8y.sh

@@ -1,7 +1,14 @@
 #!/bin/bash
 
+FILE_EXT=${FILE_EXT:-raucb}
+
+if [ $# -gt 0 ]; then
+    FILE_EXT="$1"
+    shift
+fi
+
 if ! command -V c8y >/dev/null 2>&1; then
-    echo "Missing dependency. Please install go-c8y-cli via https://goc8ycli.netlify.app/"
+    echo "Missing dependency. Please install go-c8y-cli via https://goc8ycli.netlify.app/" >&2
     exit 1
 fi
 
@@ -11,16 +18,22 @@ else
     MACHINE="${MACHINE:-raspberrypi4-64}"
 fi
 
-FIRMWARE_FILE=$(find "build/tmp/deploy/images/$MACHINE" -name "*_*.mender" | tail -1)
+SEARCH_DIR="build/tmp/deploy/images/$MACHINE"
+echo "Searching for *.$FILE_EXT files under $SEARCH_DIR" >&2
+FIRMWARE_FILE=$(find "$SEARCH_DIR" -name "*_*.$FILE_EXT" | tail -1)
 if [ -z "$FIRMWARE_FILE" ]; then
-    echo "Could not find the .mender image"
+    echo "Could not find the .$FILE_EXT image" >&2
     exit 1
 fi
 
 FIRMWARE_NAME=$(basename "$FIRMWARE_FILE" | cut -d_ -f1)
 FIRMWARE_VERSION=$(basename "$FIRMWARE_FILE" | cut -d_ -f2 | sed 's/\.[a-zA-Z]*$//g')
 
+echo "Found firmware. name=$FIRMWARE_NAME, version=$FIRMWARE_VERSION, file=$FIRMWARE_FILE" >&2
+
 # Create software name (if it does not already exist)
-c8y firmware get -n --id "$FIRMWARE_NAME" --silentStatusCodes 404 >/dev/null || c8y firmware create -n --name "$FIRMWARE_NAME" --delay "1s" --force
+c8y firmware get -n --id "$FIRMWARE_NAME" --silentStatusCodes 404 >/dev/null \
+    || c8y firmware create -n --name "$FIRMWARE_NAME" --delay "1s" --force
 
-c8y firmware versions create --file "$FIRMWARE_FILE" --firmware "$FIRMWARE_NAME" --version "$FIRMWARE_VERSION" "$@"
+c8y firmware versions get -n --firmware "$FIRMWARE_NAME" --id "$FIRMWARE_VERSION" --silentStatusCodes 404 >/dev/null \
+    || c8y firmware versions create -n --file "$FIRMWARE_FILE" --firmware "$FIRMWARE_NAME" --version "$FIRMWARE_VERSION" --force "$@"