diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3fa06e67..d3afa2c7 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,48 +1,48 @@ # Dependencies versions -ARG PYTHON_VERSION="3.9.10" +ARG PYTHON_VERSION="3.9" -# see https://hub.docker.com/_/python/?tab=tag -FROM python:${PYTHON_VERSION}-buster +FROM mcr.microsoft.com/vscode/devcontainers/python:0-${PYTHON_VERSION} # Dependencies versions -ARG AZURE_CLI_VERSION="2.24.0-1~buster" -ARG NODE_VERSION="15.x" +ARG AZURE_CLI_VERSION="2.34.1-1~bullseye" +ARG NODE_VERSION="17.x" ARG DOTNET_VERSION="5.0" -ARG JDK_VERSION="2:1.11-71" -ARG MAVEN_VERSION="3.6.0-1" +ARG JDK_VERSION="2:1.11-72" +ARG MAVEN_VERSION="3.6.3-5" -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y --no-install-recommends build-essential && \ - apt-get install -y libffi-dev libssl-dev vim +# Bring in utility scripts +COPY .devcontainer/library-scripts/*.sh /tmp/library-scripts/ -# Install python development dependencies -ADD requirements_dev.txt requirements.txt /tmp/ -RUN python -m pip install --upgrade pip -RUN pip install -r /tmp/requirements_dev.txt +RUN \ + # Install some basics + apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends build-essential apt-utils && \ + apt-get install -y libffi-dev libssl-dev vim sudo && \ + apt-get upgrade -y RUN \ - # Install the Azure CLI and IoT extension + # Upgrade to latest pip + python -m pip install --upgrade pip && \ + # Install node, npm + curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION} | bash - && \ + apt-get install -y nodejs && \ + npm install npm@latest -g && \ + # Install azure-cli apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg && \ curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null && \ echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list && \ apt-get update && apt-get install -y azure-cli=${AZURE_CLI_VERSION} && \ az extension add --name azure-iot --system && \ az extension update --name azure-iot && \ - # Install Docker CE CLI - apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common lsb-release && \ - curl -fsSL https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]')/gpg | apt-key add - 2>/dev/null && \ - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) stable" && \ - apt-get update && \ - apt-get install -y docker-ce-cli && \ - # Install node, npm, Yoeman, node.js modules - curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION} | bash - && \ - apt-get install -y nodejs && \ + # Use Docker script from script library to set things up - enable non-root docker, user vscode, using moby + /bin/bash /tmp/library-scripts/docker-in-docker-debian.sh "true" "vscode" "true" \ + # Install Yoeman, node.js modules npm install -g yo && \ npm i -g yo generator-azure-iot-edge-module && \ - cd ~ && \ # Install dotnet SDK - wget https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \ + cd ~ && \ + wget https://packages.microsoft.com/config/debian/11/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \ dpkg -i packages-microsoft-prod.deb && \ rm packages-microsoft-prod.deb && \ apt-get update && apt-get install -y apt-transport-https && \ @@ -50,19 +50,26 @@ RUN \ # Install Java JDK, Maven apt-get install -y default-jdk=${JDK_VERSION} && \ apt-get install -y maven=${MAVEN_VERSION} && \ - # + # Install bash completion + apt-get update && apt-get -y install bash-completion && \ # Clean up apt-get autoremove -y && \ apt-get clean -y && \ rm -rf /tmp/* && \ rm -rf /var/lib/apt/lists/* -# Customize bash +# customize vscode environmnet +USER vscode RUN \ - # Git command prompt - git clone https://github.com/magicmonty/bash-git-prompt.git ~/.bash-git-prompt --depth=1 && \ - echo "if [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh; fi" >> "/root/.bashrc" && \ - # Install bash completion - apt-get update && \ - apt-get -y install bash-completion && \ - echo "source /usr/share/bash-completion/bash_completion" >> ~/.bashrc + git clone https://github.com/magicmonty/bash-git-prompt.git $HOME/.bash-git-prompt --depth=1 && \ + echo "\n# setup GIT prompt and completion\nif [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then\n GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh;\nfi" >> $HOME/.bashrc && \ + echo "source /usr/share/bash-completion/bash_completion" >> $HOME/.bashrc && \ + echo "\n# add local python programs to PATH\nexport PATH=$PATH:$HOME/.local/bin\n" >> $HOME/.bashrc && \ + # enable some useful aliases + sed -i -e "s/#alias/alias/" $HOME/.bashrc && \ + # add the cookiecutter + pip install cookiecutter + +# launch docker-ce +ENTRYPOINT [ "/usr/local/share/docker-init.sh" ] +CMD [ "sleep", "infinity" ] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ed340255..cbd54f23 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,25 +4,23 @@ "name": "iotedgedev", "build": { "dockerfile": "Dockerfile", - "context": "..", + "context": ".." }, - "runArgs": [ - "--network", - "host" - ], - - "postCreateCommand": "pip install -e .", + "runArgs": ["--init", "--privileged"], "mounts": [ // Keep command history "source=ostf-bashhistory,target=/commandhistory,type=volume", - // Use hosts docker socket - "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind", + // Use docker-in-docker socket + "source=dind-var-lib-docker,target=/var/lib/docker,type=volume" ], + "overrideCommand": false, + "postCreateCommand": "docker image prune -a -f && sudo chown vscode:users -R /home/vscode/Dev && pip install -r requirements_dev.txt && pip install -e .", + // Set *default* container specific settings.json values on container create. "settings": { - "#terminal.integrated.defaultProfile.linux#": "/bin/bash", + "#terminal.integrated.defaultProfile.linux#": "/bin/bash" }, // Add the IDs of extensions you want installed when the container is created. @@ -34,5 +32,9 @@ "streetsidesoftware.code-spell-checker", "yzhang.markdown-all-in-one", "davidanson.vscode-markdownlint" - ] + ], + + "workspaceMount": "source=${localWorkspaceFolder},target=/home/vscode/Dev,type=bind,consistency=cached", + "workspaceFolder": "/home/vscode/Dev", + "remoteUser": "vscode" } diff --git a/.devcontainer/library-scripts/docker-in-docker-debian.sh b/.devcontainer/library-scripts/docker-in-docker-debian.sh new file mode 100644 index 00000000..bcb63ffa --- /dev/null +++ b/.devcontainer/library-scripts/docker-in-docker-debian.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. +#------------------------------------------------------------------------------------------------------------- +# +# Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/docker-in-docker.md +# Maintainer: The VS Code and Codespaces Teams +# +# Syntax: ./docker-in-docker-debian.sh [enable non-root docker access flag] [non-root user] [use moby] + +ENABLE_NONROOT_DOCKER=${1:-"true"} +USERNAME=${2:-"automatic"} +USE_MOBY=${3:-"true"} +MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" + +set -e + +if [ "$(id -u)" -ne 0 ]; then + echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' + exit 1 +fi + +# Determine the appropriate non-root user +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in ${POSSIBLE_USERS[@]}; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi +elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root +fi + +# Get central common setting +get_common_setting() { + if [ "${common_settings_file_loaded}" != "true" ]; then + curl -sfL "https://aka.ms/vscode-dev-containers/script-library/settings.env" 2>/dev/null -o /tmp/vsdc-settings.env || echo "Could not download settings file. Skipping." + common_settings_file_loaded=true + fi + if [ -f "/tmp/vsdc-settings.env" ]; then + local multi_line="" + if [ "$2" = "true" ]; then multi_line="-z"; fi + local result="$(grep ${multi_line} -oP "$1=\"?\K[^\"]+" /tmp/vsdc-settings.env | tr -d '\0')" + if [ ! -z "${result}" ]; then declare -g $1="${result}"; fi + fi + echo "$1=${!1}" +} + +# Function to run apt-get if needed +apt_get_update_if_needed() +{ + if [ ! -d "/var/lib/apt/lists" ] || [ "$(ls /var/lib/apt/lists/ | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update + else + echo "Skipping apt-get update." + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update_if_needed + apt-get -y install --no-install-recommends "$@" + fi +} + +# Ensure apt is in non-interactive to avoid prompts +export DEBIAN_FRONTEND=noninteractive + +# Install dependencies +check_packages apt-transport-https curl ca-certificates lxc pigz iptables gnupg2 dirmngr + +# Swap to legacy iptables for compatibility +if type iptables-legacy > /dev/null 2>&1; then + update-alternatives --set iptables /usr/sbin/iptables-legacy + update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy +fi + +# Install Docker / Moby CLI if not already installed +architecture="$(dpkg --print-architecture)" +if type docker > /dev/null 2>&1 && type dockerd > /dev/null 2>&1; then + echo "Docker / Moby CLI and Engine already installed." +else + # Source /etc/os-release to get OS info + . /etc/os-release + if [ "${USE_MOBY}" = "true" ]; then + # Import key safely (new 'signed-by' method rather than deprecated apt-key approach) and install + get_common_setting MICROSOFT_GPG_KEYS_URI + curl -sSL ${MICROSOFT_GPG_KEYS_URI} | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg + echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list + apt-get update + apt-get -y install --no-install-recommends moby-cli moby-buildx moby-engine + apt-get -y install --no-install-recommends moby-compose || echo "(*) Package moby-compose (Docker Compose v2) not available for ${VERSION_CODENAME} ${architecture}. Skipping." + else + # Import key safely (new 'signed-by' method rather than deprecated apt-key approach) and install + curl -fsSL https://download.docker.com/linux/${ID}/gpg | gpg --dearmor > /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/${ID} ${VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list + apt-get update + apt-get -y install --no-install-recommends docker-ce-cli docker-ce + fi +fi + +echo "Finished installing docker / moby" + +# Install Docker Compose if not already installed and is on a supported architecture +if type docker-compose > /dev/null 2>&1; then + echo "Docker Compose already installed." +else + target_compose_arch="${architecture}" + if [ "${target_compose_arch}" != "x86_64" ]; then + # Use pip to get a version that runns on this architecture + if ! dpkg -s python3-minimal python3-pip libffi-dev python3-venv > /dev/null 2>&1; then + apt_get_update_if_needed + apt-get -y install python3-minimal python3-pip libffi-dev python3-venv + fi + export PIPX_HOME=/usr/local/pipx + mkdir -p ${PIPX_HOME} + export PIPX_BIN_DIR=/usr/local/bin + export PYTHONUSERBASE=/tmp/pip-tmp + export PIP_CACHE_DIR=/tmp/pip-tmp/cache + pipx_bin=pipx + if ! type pipx > /dev/null 2>&1; then + pip3 install --disable-pip-version-check --no-warn-script-location --no-cache-dir --user pipx + pipx_bin=/tmp/pip-tmp/bin/pipx + fi + ${pipx_bin} install --system-site-packages --pip-args '--no-cache-dir --force-reinstall' docker-compose + rm -rf /tmp/pip-tmp + else + latest_compose_version=$(basename "$(curl -fsSL -o /dev/null -w "%{url_effective}" https://github.com/docker/compose/releases/latest)") + curl -fsSL "https://github.com/docker/compose/releases/download/${latest_compose_version}/docker-compose-$(uname -s)-${target_compose_arch}" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + fi +fi + +# If init file already exists, exit +if [ -f "/usr/local/share/docker-init.sh" ]; then + echo "/usr/local/share/docker-init.sh already exists, so exiting." + exit 0 +fi +echo "docker-init doesnt exist..." + +# Add user to the docker group +if [ "${ENABLE_NONROOT_DOCKER}" = "true" ]; then + if ! getent group docker > /dev/null 2>&1; then + groupadd docker + fi + + usermod -aG docker ${USERNAME} +fi + +tee /usr/local/share/docker-init.sh > /dev/null \ +<< 'EOF' +#!/usr/bin/env bash +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. +#------------------------------------------------------------------------------------------------------------- + +sudoIf() +{ + if [ "$(id -u)" -ne 0 ]; then + sudo "$@" + else + "$@" + fi +} + +# explicitly remove dockerd and containerd PID file to ensure that it can start properly if it was stopped uncleanly +# ie: docker kill +sudoIf find /run /var/run -iname 'docker*.pid' -delete || : +sudoIf find /run /var/run -iname 'container*.pid' -delete || : + +set -e + +## Dind wrapper script from docker team +# Maintained: https://github.com/moby/moby/blob/master/hack/dind + +export container=docker + +if [ -d /sys/kernel/security ] && ! sudoIf mountpoint -q /sys/kernel/security; then + sudoIf mount -t securityfs none /sys/kernel/security || { + echo >&2 'Could not mount /sys/kernel/security.' + echo >&2 'AppArmor detection and --privileged mode might break.' + } +fi + +# Mount /tmp (conditionally) +if ! sudoIf mountpoint -q /tmp; then + sudoIf mount -t tmpfs none /tmp +fi + +# cgroup v2: enable nesting +if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + # move the init process (PID 1) from the root group to the /init group, + # otherwise writing subtree_control fails with EBUSY. + sudoIf mkdir -p /sys/fs/cgroup/init + sudoIf echo 1 > /sys/fs/cgroup/init/cgroup.procs + # enable controllers + sudoIf sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control +fi +## Dind wrapper over. + +# Handle DNS +set +e +cat /etc/resolv.conf | grep -i 'internal.cloudapp.net' +if [ $? -eq 0 ] +then + echo "Setting dockerd Azure DNS." + CUSTOMDNS="--dns 168.63.129.16" +else + echo "Not setting dockerd DNS manually." + CUSTOMDNS="" +fi +set -e + +# Start docker/moby engine +( sudoIf dockerd $CUSTOMDNS > /tmp/dockerd.log 2>&1 ) & + +set +e + +# Execute whatever commands were passed in (if any). This allows us +# to set this script to ENTRYPOINT while still executing the default CMD. +exec "$@" +EOF + +chmod +x /usr/local/share/docker-init.sh +chown ${USERNAME}:root /usr/local/share/docker-init.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 5575ed00..8acd19ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ All notable changes to this project since 0.82.0 will be documented in this file ## [3.3.4] - 2022-03-24 - Lock simulator version at 0.14.14 +- Update Wiki and dev container to reflect changes about the simulator ## [3.3.3] - 2022-03-15 - Add support for python 3.8, 3.9 diff --git a/docs/Manual-Dev-Machine-Setup.md b/docs/Manual-Dev-Machine-Setup.md index 6dd2eff0..b1b5668a 100644 --- a/docs/Manual-Dev-Machine-Setup.md +++ b/docs/Manual-Dev-Machine-Setup.md @@ -20,8 +20,8 @@ If you are using a separate Edge device, like a Raspberry Pi, you do not need to 1. Install **[Azure CLI IoT extension](https://github.com/Azure/azure-iot-cli-extension/)** - - New Install: `az extension add --name azure-cli-iot-ext` - - Update Install: `az extension update --name azure-cli-iot-ext` + - New Install: `az extension add --name azure-iot` + - Update Install: `az extension update --name azure-iot` 1. (Python < 3.5 only) Install **[Node.js](https://nodejs.org/en/download/)** and the **`iothub-explorer`** package @@ -74,4 +74,4 @@ If you are using a separate Edge device, like a Raspberry Pi, you do not need to 1. Install **[JDK](https://www.oracle.com/technetwork/java/javase/downloads/index.html)** 1. Install **[Maven](https://maven.apache.org/)** -1. Follow the [Usage Wiki](usage) to learn the usage of IoT Edge Dev Tool \ No newline at end of file +1. Follow the [Usage Wiki](usage) to learn the usage of IoT Edge Dev Tool diff --git a/requirements_dev.txt b/requirements_dev.txt index 4b0964a6..04d5669d 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -3,7 +3,7 @@ bumpversion==0.5.3 wheel==0.30.0 watchdog==0.8.3 docker-compose==1.29.1 -flake8==3.9.2 +flake8 pycodestyle==2.7.0 autopep8==1.5.7 tox==3.24.1