From 6568dff8e1e196e7774448e58801d7bd8a4c3a08 Mon Sep 17 00:00:00 2001 From: Stefan de Kooter Date: Wed, 6 Nov 2019 10:26:29 +0100 Subject: [PATCH 01/40] Add SSLMODE to database connection properties Defaulting to SSLMODE=prefer, optional override using environment variable: DB_SSLMODE --- configuration/configuration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configuration/configuration.py b/configuration/configuration.py index 14c5d83..23170e8 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -37,6 +37,8 @@ DATABASE = { # PostgreSQL password 'HOST': os.environ.get('DB_HOST', 'localhost'), # Database server 'PORT': os.environ.get('DB_PORT', ''), # Database port (leave blank for default) + 'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')}, + # Database connection SSLMODE } # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. From f2fd7dbbe7e6d9f1d2a36a8f14abaee17a6ede71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Wed, 20 Nov 2019 11:09:37 +0100 Subject: [PATCH 02/40] Adds a link to the Getting Started wiki section to the README --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e375bea..5c0ff33 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,15 @@ $ xdg-open "http://$(docker-compose port nginx 8080)/" &>/dev/null & Alternatively, use something like [Reception][docker-reception] to connect to _docker-compose_ projects. -Default credentials: +The default credentials are: * Username: **admin** * Password: **admin** * API Token: **0123456789abcdef0123456789abcdef01234567** +There is a more complete [Getting Started guide on our Wiki][wiki-getting-started]. + +[wiki-getting-started]: https://github.com/netbox-community/netbox-docker/wiki/Getting-Started [docker-reception]: https://github.com/nxt-engineering/reception ## Dependencies From 01c4137dc9d6e3f8e0e44f56328222d994d27e3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 17 Nov 2019 18:03:59 +0100 Subject: [PATCH 03/40] Adds netbox user --- Dockerfile | 10 ++++++++++ docker-compose.yml | 1 + 2 files changed, 11 insertions(+) diff --git a/Dockerfile b/Dockerfile index d7a9af9..da15ecb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,10 @@ RUN apk add --no-cache \ postgresql-libs \ ttf-ubuntu-font-family +RUN addgroup -S -g 101 netbox \ + && adduser -DHS -u 101 netbox \ + && adduser netbox netbox + WORKDIR /opt COPY --from=builder /install /usr/local @@ -68,6 +72,12 @@ COPY configuration/configuration.py /etc/netbox/config/configuration.py WORKDIR /opt/netbox/netbox +# Must set permissions for '/opt/netbox/netbox/static' directory +# to a+w so that `./manage.py collectstatic` can be executed during +# container startup. +# Not satisfying +RUN mkdir static && chmod a+w static media + ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"] diff --git a/docker-compose.yml b/docker-compose.yml index f00700a..da5fbd5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: - redis - netbox-worker env_file: env/netbox.env + user: netbox volumes: - ./startup_scripts:/opt/netbox/startup_scripts:z,ro - ./initializers:/opt/netbox/initializers:z,ro From 7942e9edbe89469fd691d1bed634ab84decb00b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 18 Nov 2019 08:23:44 +0100 Subject: [PATCH 04/40] Remove the named user, change permissions to 'g+w' --- Dockerfile | 11 ++++------- docker-compose.yml | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index da15ecb..af70e38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,10 +51,6 @@ RUN apk add --no-cache \ postgresql-libs \ ttf-ubuntu-font-family -RUN addgroup -S -g 101 netbox \ - && adduser -DHS -u 101 netbox \ - && adduser netbox netbox - WORKDIR /opt COPY --from=builder /install /usr/local @@ -73,10 +69,11 @@ COPY configuration/configuration.py /etc/netbox/config/configuration.py WORKDIR /opt/netbox/netbox # Must set permissions for '/opt/netbox/netbox/static' directory -# to a+w so that `./manage.py collectstatic` can be executed during +# to g+w so that `./manage.py collectstatic` can be executed during # container startup. -# Not satisfying -RUN mkdir static && chmod a+w static media +# Must set permissions for '/opt/netbox/netbox/media' directory +# to g+w so that pictures can be uploaded to netbox. +RUN mkdir static && chmod g+w static media ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] diff --git a/docker-compose.yml b/docker-compose.yml index da5fbd5..874ff3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: - redis - netbox-worker env_file: env/netbox.env - user: netbox + user: '101' volumes: - ./startup_scripts:/opt/netbox/startup_scripts:z,ro - ./initializers:/opt/netbox/initializers:z,ro From 8664e42233175ce2a2e9b78163ae01c0de26b789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 11:40:10 +0100 Subject: [PATCH 05/40] Revert "Fix #179 'libc not found'" This reverts commit feb810ab27bf459a54a3e2055e7c399ed478f1f5. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d7a9af9..d96fd01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ WORKDIR /install RUN pip install --prefix="/install" --no-warn-script-location \ # gunicorn is used for launching netbox - 'gunicorn<20.0.0' \ + gunicorn \ greenlet \ eventlet \ # napalm is used for gathering information from network devices From 5e92352b0ae83f2c6a065ea31520300d40a3e732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 12:09:26 +0100 Subject: [PATCH 06/40] set umask in entrypoint --- docker/docker-entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 2e6a8de..7aef836 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +umask 002 # wait shortly and then run db migrations (retry on error) while ! ./manage.py migrate 2>&1; do From 1ce99cf02f2e28bc26f7fb66352714df969d8cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 12:14:51 +0100 Subject: [PATCH 07/40] update redis --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f00700a..e00f8c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,7 +41,7 @@ services: volumes: - netbox-postgres-data:/var/lib/postgresql/data redis: - image: redis:4-alpine + image: redis:5-alpine command: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env From 150f35ea3bc8e3312d48e9fe97b68e86671dc6aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 12:23:09 +0100 Subject: [PATCH 08/40] update postgres to 12 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f00700a..f81f325 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: - netbox-static-files:/opt/netbox/netbox/static:ro - netbox-nginx-config:/etc/netbox-nginx/:ro postgres: - image: postgres:10.4-alpine + image: postgres:12-alpine env_file: env/postgres.env volumes: - netbox-postgres-data:/var/lib/postgresql/data From 788804627e29a125c9f5668782e3bcf96e6f29ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 12:27:25 +0100 Subject: [PATCH 09/40] Fix typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5c0ff33..a0f1fee 100644 --- a/README.md +++ b/README.md @@ -50,12 +50,12 @@ There is a more complete [Getting Started guide on our Wiki][wiki-getting-starte ## Dependencies -This project relies only on *Docker* and *docker-compose* meeting this requirements: +This project relies only on *Docker* and *docker-compose* meeting these requirements: * The *Docker version* must be at least `17.05`. * The *docker-compose version* must be at least `1.17.0`. -To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above. +To check the version installed on your system, compare the output of `docker --version` and `docker-compose --version` with the requirements above. ## Reference Documentation From 8a92640d1001fb074849a98d8d9b725461f322c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 26 Nov 2019 12:28:43 +0100 Subject: [PATCH 10/40] Re-wording of dependencies check paragraph --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a0f1fee..82f01aa 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ This project relies only on *Docker* and *docker-compose* meeting these requirem * The *Docker version* must be at least `17.05`. * The *docker-compose version* must be at least `1.17.0`. -To check the version installed on your system, compare the output of `docker --version` and `docker-compose --version` with the requirements above. +To check the version installed on your system run `docker --version` and `docker-compose --version`. ## Reference Documentation From 809570f4bc5db6b0e54e5dbbada1d6966f44165b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 10 Dec 2019 21:25:36 +0100 Subject: [PATCH 11/40] Fix variables for Dockerhub build --- hooks/common | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hooks/common b/hooks/common index 49cd507..0f0223e 100755 --- a/hooks/common +++ b/hooks/common @@ -14,7 +14,7 @@ ensure_jq() { # Passes args to the scripts run_build() { echo "đŸŗđŸŗđŸŗ Building '${BUILD}' images" - case $BUILD in + case ${BUILD} in release) # build the latest release # shellcheck disable=SC2068 @@ -49,9 +49,9 @@ run_build() { } echo "🤖🤖🤖 Preparing build" -export DOCKER_ORG="index.docker.io/netboxcommunity" +export DOCKER_ORG=netboxcommunity export DOCKER_REPO=netbox -export DOCKERHUB_REPO=netboxcommunity/netbox +export DOCKER_REGISTRY=docker.io # shellcheck disable=SC2153 export BUILD="${DOCKER_TAG}" From a4186c1031e031d89e1d78dcce417b629f084b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 10 Dec 2019 21:44:11 +0100 Subject: [PATCH 12/40] Prepares scripts and documentation for #184 --- DOCKER_HUB.md | 6 +++--- README.md | 4 ++-- build-branches.sh | 2 +- build.sh | 20 ++++++++++---------- hooks/common | 2 +- hooks/push | 4 ++-- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/DOCKER_HUB.md b/DOCKER_HUB.md index 10c4154..d3b3e68 100644 --- a/DOCKER_HUB.md +++ b/DOCKER_HUB.md @@ -11,21 +11,21 @@ Autotest: Internal and External Pull Requests Repository Links: Enable for Base Image Build Rules: - Source Type: Branch - Source: master + Source: release Docker Tag: branches Dockerfile location: Dockerfile Build Context: / Autobuild: on Build Caching: on - Source Type: Branch - Source: master + Source: release Docker Tag: prerelease Dockerfile location: Dockerfile Build Context: / Autobuild: on Build Caching: on - Source Type: Branch - Source: master + Source: release Docker Tag: release Dockerfile location: Dockerfile Build Context: / diff --git a/README.md b/README.md index 13f7241..00f3aaf 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,10 @@ Then there is currently one extra tags for each of the above labels: ## Quickstart -To get Netbox up and running: +To get Netbox up and running in Docker: ```bash -git clone -b master https://github.com/netbox-community/netbox-docker.git +git clone -b release https://github.com/netbox-community/netbox-docker.git cd netbox-docker docker-compose pull docker-compose up -d diff --git a/build-branches.sh b/build-branches.sh index a6bc736..483e771 100755 --- a/build-branches.sh +++ b/build-branches.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Builds develop, develop-* and master branches +# Builds develop, develop-* and master branches of Netbox echo "â–ļī¸ $0 $*" diff --git a/build.sh b/build.sh index 14ab409..8ea1476 100755 --- a/build.sh +++ b/build.sh @@ -27,9 +27,9 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " Default: undefined" echo " TAG The version part of the docker tag." echo " Default:" - echo " When \${BRANCH}=master: latest" - echo " When \${BRANCH}=develop: snapshot" - echo " Else: same as \${BRANCH}" + echo " When =master: latest" + echo " When =develop: snapshot" + echo " Else: same as " echo " DOCKER_REGISTRY The Docker repository's registry (i.e. '\${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}'')" echo " Used for tagging the image." echo " Default: docker.io" @@ -106,7 +106,7 @@ fi ### SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" -BRANCH="${1}" +NETBOX_BRANCH="${1}" URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" NETBOX_PATH="${NETBOX_PATH-.netbox}" @@ -114,9 +114,9 @@ NETBOX_PATH="${NETBOX_PATH-.netbox}" # fetching the source ### if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then - echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" + echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" if [ ! -d "${NETBOX_PATH}" ]; then - $DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}" + $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}" fi ( @@ -127,7 +127,7 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then fi $DRY git remote set-url origin "${URL}" - $DRY git fetch -qp --depth 10 origin "${BRANCH}" + $DRY git fetch -qp --depth 10 origin "${NETBOX_BRANCH}" $DRY git checkout -qf FETCH_HEAD $DRY git prune ) @@ -174,13 +174,13 @@ fi DOCKER_REGISTRY="${DOCKER_REGISTRY-docker.io}" DOCKER_ORG="${DOCKER_ORG-netboxcommunity}" DOCKER_REPO="${DOCKER_REPO-netbox}" -case "${BRANCH}" in +case "${NETBOX_BRANCH}" in master) TAG="${TAG-latest}";; develop) TAG="${TAG-snapshot}";; *) - TAG="${TAG-$BRANCH}";; + TAG="${TAG-$NETBOX_BRANCH}";; esac ### @@ -241,7 +241,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do if [ "${DOCKER_TARGET}" == "main" ]; then DOCKER_BUILD_ARGS+=( --label "ORIGINAL_TAG=${TARGET_DOCKER_TAG}" - + --label "org.label-schema.build-date=${BUILD_DATE}" --label "org.opencontainers.image.created=${BUILD_DATE}" diff --git a/hooks/common b/hooks/common index 49cd507..c0ad1a8 100755 --- a/hooks/common +++ b/hooks/common @@ -31,7 +31,7 @@ run_build() { ./build-branches.sh $@ ;; this) # Pull Requests - # only build the 'master' branch + # only build the 'master' branch of netbox # (resulting in the 'latest' docker tag) # and the 'main' target. DOCKER_TARGET=main ./build.sh master diff --git a/hooks/push b/hooks/push index 81dc7db..6513b24 100755 --- a/hooks/push +++ b/hooks/push @@ -2,8 +2,8 @@ . hooks/common -if [ "${SOURCE_BRANCH}" == "master" ] || [ "${DEBUG}" == "true" ]; then - if [ "${SOURCE_BRANCH}" != "master" ]; then +if [ "${SOURCE_BRANCH}" == "release" ] || [ "${DEBUG}" == "true" ]; then + if [ "${SOURCE_BRANCH}" != "release" ]; then echo "⚠ī¸âš ī¸âš ī¸ Would exit, but DEBUG is '${DEBUG}'". fi From e4e0a63e174e695febae5100ed67e65ffc097072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 10 Dec 2019 22:41:57 +0100 Subject: [PATCH 13/40] Better safe than sorry, keep PostgreSQL at 11 for now. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f81f325..d6a0109 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: - netbox-static-files:/opt/netbox/netbox/static:ro - netbox-nginx-config:/etc/netbox-nginx/:ro postgres: - image: postgres:12-alpine + image: postgres:11-alpine env_file: env/postgres.env volumes: - netbox-postgres-data:/var/lib/postgresql/data From 7df5da38bf69f6639219918102f14fc2801b4d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 10 Dec 2019 23:09:43 +0100 Subject: [PATCH 14/40] Pull request template suggestion --- .github/ISSUE_TEMPLATE/bug_report.md | 2 ++ .github/ISSUE_TEMPLATE/feature_request.md | 4 ++- .github/pull_request_template.md | 44 +++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 .github/pull_request_template.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a4771fa..7717abd 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -25,6 +25,8 @@ Please try this means to get help before opening an issue here: * On the networktocode Slack in the #netbox channel: http://slack.networktocode.com/ * On the Netbox mailing list: https://groups.google.com/d/forum/netbox-discuss +Please don't open an issue when you have a PR ready. Just submit the PR, that's good enough. + --> ## Current Behavior diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 65f31dd..6bc6dd4 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -24,6 +24,8 @@ Please try this means to get help before opening an issue here: * On the networktocode Slack in the #netbox channel: http://slack.networktocode.com/ * On the Netbox mailing list: https://groups.google.com/d/forum/netbox-discuss +Please don't open an issue when you have a PR ready. Just submit the PR, that's good enough. + --> ## Desired Behavior @@ -33,7 +35,7 @@ Please try this means to get help before opening an issue here: ## Contrast to Current Behavior - + ... ## Changes Required diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..9d85010 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,44 @@ + +Related Issue: + +## New Behavior + + + +... + +## Contrast to Current Behavior + + + +... + +## Discussion: Benefits and Drawbacks + + + +... + +## Changes to the Wiki + + + +... From 1064696c96773ac8b1602757bdc24d990b168545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Tue, 10 Dec 2019 23:16:11 +0100 Subject: [PATCH 15/40] N/A when there's no issue --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9d85010..1c3885f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,7 +1,7 @@ +But if there is already a related issue, please put it's number here. E.g. #123 or N/A --> Related Issue: ## New Behavior From 93dee744599b520beff6412939216a3b95104aae Mon Sep 17 00:00:00 2001 From: Jeroen Simonetti Date: Mon, 2 Dec 2019 10:23:53 +0100 Subject: [PATCH 16/40] Add rack group initialiser Fixes #192 Signed-off-by: Jeroen Simonetti --- initializers/rack_groups.yml | 3 +++ initializers/racks.yml | 1 + startup_scripts/075_rack_groups.py | 31 ++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+) create mode 100644 initializers/rack_groups.yml create mode 100644 startup_scripts/075_rack_groups.py diff --git a/initializers/rack_groups.yml b/initializers/rack_groups.yml new file mode 100644 index 0000000..244fc00 --- /dev/null +++ b/initializers/rack_groups.yml @@ -0,0 +1,3 @@ +# - name: cage 101 +# slug: cage-101 +# site: SING 1 diff --git a/initializers/racks.yml b/initializers/racks.yml index 9a71743..3a31526 100644 --- a/initializers/racks.yml +++ b/initializers/racks.yml @@ -16,6 +16,7 @@ # text_field: Description # - site: SING 1 # name: rack-03 +# group: cage 101 # role: Role 3 # type: 4-post cabinet # width: 19 inches diff --git a/startup_scripts/075_rack_groups.py b/startup_scripts/075_rack_groups.py new file mode 100644 index 0000000..7deaa11 --- /dev/null +++ b/startup_scripts/075_rack_groups.py @@ -0,0 +1,31 @@ +from dcim.models import Site,RackGroup +from ruamel.yaml import YAML + +from pathlib import Path +import sys + +file = Path('/opt/netbox/initializers/rack_groups.yml') +if not file.is_file(): + sys.exit() + +with file.open('r') as stream: + yaml=YAML(typ='safe') + rack_groups= yaml.load(stream) + + required_assocs = { + 'site': (Site, 'name') + } + + if rack_groups is not None: + for params in rack_groups: + + for assoc, details in required_assocs.items(): + model, field = details + query = { field: params.pop(assoc) } + params[assoc] = model.objects.get(**query) + + rack_group, created = RackGroup.objects.get_or_create(**params) + + if created: + print("🎨 Created rack group", rack_group.name) + From daaf72962f8e357272dcca37b2e9219e83b9ae6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Wed, 11 Dec 2019 11:48:29 +0100 Subject: [PATCH 17/40] Adds a test that validates the initializer yml files --- hooks/test | 13 +++---------- test.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 10 deletions(-) create mode 100755 test.sh diff --git a/hooks/test b/hooks/test index 1dd5538..a66db12 100755 --- a/hooks/test +++ b/hooks/test @@ -2,21 +2,14 @@ . hooks/common -run_test() { - echo "đŸŗđŸŗđŸŗ Testing '${1}'" - VERSION="${1}" docker-compose run netbox ./manage.py test - docker-compose down -v - echo "đŸŗđŸŗđŸŗ Done testing '${1}'" -} - # test on builds of 'branches' if [ "${BUILD}" == "branches" ] \ || [ "${DEBUG}" == "true" ]; then - run_test latest - run_test snapshot + ./test.sh latest + ./test.sh snapshot # test on bulds of 'this' (i.e. pull request) elif [ "${BUILD}" == "this" ]; then - run_test latest + ./test.sh latest else echo "đŸŗđŸŗđŸŗ No tests are implemented for build '${BUILD}'." fi diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..d6d784a --- /dev/null +++ b/test.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# exit when a command exits with an exit code != 0 +set -e + +# version is used by `docker-compose.yml` do determine the tag +# of the Docker Image that is to be used +export VERSION=${VERSION-latest} + +test_netbox_unit_tests() { + echo "⏱ Running Netbox Unit Tests" + docker-compose run --rm netbox ./manage.py test +} + +test_initializers() { + echo "🏗 Testing Initializers" + + mkdir initializers_test + ( + cd initializers + for script in *.yml; do + sed -E 's/^# //' "${script}" > "../initializers_test/${script}" + done + ) + mv initializers initializers_original + mv initializers_test initializers + + docker-compose run --rm netbox ./manage.py check +} + +test_cleanup() { + echo "đŸ’Ŗ Cleaning Up" + docker-compose down -v + rm -rf initializers + mv initializers_original initializers +} + +echo "đŸŗđŸŗđŸŗ Start testing '${VERSION}'" + +# Make sure the cleanup script is executed +trap test_cleanup EXIT ERR + +test_netbox_unit_tests +test_initializers + +echo "đŸŗđŸŗđŸŗ Done testing '${VERSION}'" From 310cda1f185cd9457b2dd7b3e44610d785364bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Wed, 11 Dec 2019 13:58:33 +0100 Subject: [PATCH 18/40] Reduce the Docker Context --- .dockerignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.dockerignore b/.dockerignore index 23492a4..bdb18c4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,7 @@ env build* docker-compose.override.yml +.netbox/.git* +.netbox/.travis.yml +.netbox/docs +.netbox/scripts From 66e90428b52f9f8dabf35136aa0cbef29f1c0491 Mon Sep 17 00:00:00 2001 From: Matt Olenik Date: Mon, 2 Dec 2019 12:17:40 -0800 Subject: [PATCH 19/40] Remove use of GNU date extensions when labeling images Fixes #197 --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 14ab409..104bb99 100755 --- a/build.sh +++ b/build.sh @@ -152,7 +152,7 @@ fi ### # variables for labelling the docker image ### -BUILD_DATE="$(date --utc --iso-8601=minutes)" +BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M+00:00')" if [ -d ".git" ]; then GIT_REF="$(git rev-parse HEAD)" From 05d32ae70537b590b08cf0c533a1ee20e8032563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sat, 14 Dec 2019 18:16:31 +0100 Subject: [PATCH 20/40] Massive speedup in executing startup_scripts --- docker/docker-entrypoint.sh | 5 +---- startup_scripts/__main__.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 startup_scripts/__main__.py diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 2e6a8de..0c8d2a1 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -45,10 +45,7 @@ fi if [ "$SKIP_STARTUP_SCRIPTS" == "true" ]; then echo "↩ī¸ Skipping startup scripts" else - for script in /opt/netbox/startup_scripts/*.py; do - echo "⚙ī¸ Executing '$script'" - ./manage.py shell --interface python < "${script}" - done + echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python fi # copy static files diff --git a/startup_scripts/__main__.py b/startup_scripts/__main__.py new file mode 100644 index 0000000..50d2613 --- /dev/null +++ b/startup_scripts/__main__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import runpy +from os import scandir +from os.path import dirname, abspath + +this_dir = dirname(abspath(__file__)) + +def filename(f): + return f.name + +with scandir(dirname(abspath(__file__))) as it: + for f in sorted(it, key = filename): + if f.name.startswith('__') or not f.is_file(): + continue + + print(f"Running {f.path}") + runpy.run_path(f.path) From 7863e5902e55ca06f9a4c6f9e1cfc0c98aee3709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sat, 21 Dec 2019 15:20:33 +0100 Subject: [PATCH 21/40] Preparation for 0.20.0 --- README.md | 6 +++--- VERSION | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ed04478..363980c 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ To use this feature, set the environment-variable `VERSION` before launching `do [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub]. ```bash -export VERSION=v2.6.7 +export VERSION=v2.6.9 docker-compose pull netbox docker-compose up -d ``` @@ -112,7 +112,7 @@ You can also build a specific version of the Netbox Docker image yourself. `VERSION` can be any valid [git ref][git-ref] in that case. ```bash -export VERSION=v2.6.7 +export VERSION=v2.6.9 ./build.sh $VERSION docker-compose up -d ``` @@ -126,7 +126,7 @@ From time to time it might become necessary to re-engineer the structure of this Things like the `docker-compose.yml` file or your Kubernetes or OpenShift configurations have to be adjusted as a consequence. Since November 2019 each image built from this repo contains a `org.opencontainers.image.version` label. (The images contained labels since April 2018, although in November 2019 the labels' names changed.) -You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.6.7 --format "{{json .ContainerConfig.Labels}}"`. +You can check the label of your local image by running `docker inspect netboxcommunity/netbox:v2.6.9 --format "{{json .ContainerConfig.Labels}}"`. Please read [the release notes][releases] carefully when updating to a new image version. diff --git a/VERSION b/VERSION index c0b8d59..5a03fb7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.19.4 +0.20.0 From b118cd58127c71c3b2c346926e9728837b98bb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 23 Dec 2019 17:53:19 +0100 Subject: [PATCH 22/40] Building the Docker image with Github Actions --- .github/workflows/push.yml | 31 ++++++++++++++++++ .github/workflows/release.yml | 39 +++++++++++++++++++++++ build.sh | 20 ++++++++---- docker-compose.test.yml | 45 ++++++++++++++++++++++++++ docker-compose.yml | 2 -- docker/docker-entrypoint.sh | 27 ++++++++++++---- hooks/build | 6 ---- hooks/common | 60 ----------------------------------- hooks/push | 14 -------- hooks/test | 15 --------- test.sh | 22 ++++++++----- 11 files changed, 164 insertions(+), 117 deletions(-) create mode 100644 .github/workflows/push.yml create mode 100644 .github/workflows/release.yml create mode 100644 docker-compose.test.yml delete mode 100755 hooks/build delete mode 100755 hooks/common delete mode 100755 hooks/push delete mode 100755 hooks/test diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..3290e27 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,31 @@ +on: + push: + branches-ignore: + - release + +jobs: + build: + strategy: + matrix: + build_cmd: + - ./build-latest.sh + - PRERELEASE=true ./build-latest.sh + - ./build-branches.sh + docker_from: + - '' # use the default of the DOCKERFILE + - python:3.7-alpine + - python:3.8-alpine + - python:3.9-rc-alpine + fail-fast: false + runs-on: ubuntu-latest + name: Builds new Netbox Docker Images + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}' + run: ${{ matrix.build_cmd }} + env: + DOCKER_FROM: ${{ matrix.docker_from }} + GH_ACTION: enable + - name: Test the image + run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..72a0bdb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,39 @@ +on: + push: + branches: + - release + schedule: + - cron: '45 5 * * *' + +jobs: + build: + strategy: + matrix: + build_cmd: + - ./build-latest.sh + - PRERELEASE=true ./build-latest.sh + - ./build-branches.sh + fail-fast: false + runs-on: ubuntu-latest + name: Builds new Netbox Docker Images + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Build the image with '${{ matrix.build_cmd }}' + run: ${{ matrix.build_cmd }} + env: + GH_ACTION: enable + - name: Test the image + run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh + - name: Login to the Docker Registry + run: | + echo "::add-mask::$DOCKERHUB_USERNAME" + echo "::add-mask::$DOCKERHUB_PASSWORD" + docker login -u "$DOCKERHUB_USERNAME" --password "${DOCKERHUB_PASSWORD}" "${DOCKER_REGISTRY}" + env: + DOCKERHUB_USERNAME: ${{ secrets.dockerhub_username }} + DOCKERHUB_PASSWORD: ${{ secrets.dockerhub_password }} + - name: Push the image + run: ${{ matrix.build_cmd }} --push-only + - name: Logout of the Docker Registry + run: docker logout "${DOCKER_REGISTRY}" diff --git a/build.sh b/build.sh index 01e841a..94cbe6a 100755 --- a/build.sh +++ b/build.sh @@ -63,6 +63,10 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " Default: undefined" echo " DRY_RUN Prints all build statements instead of running them." echo " Default: undefined" + echo " GH_ACTION If defined, special 'echo' statements are enabled that set the" + echo " following environment variables in Github Actions:" + echo " - FINAL_DOCKER_TAG: The final value of the DOCKER_TAG env variable" + echo " Default: undefined" echo "" echo "Examples:" echo " ${0} master" @@ -92,7 +96,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then fi ### -# Determining the build command to use +# Enabling dry-run mode ### if [ -z "${DRY_RUN}" ]; then DRY="" @@ -102,7 +106,7 @@ else fi ### -# variables for fetching the source +# Variables for fetching the source ### SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" @@ -111,7 +115,7 @@ URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" NETBOX_PATH="${NETBOX_PATH-.netbox}" ### -# fetching the source +# Fetching the source ### if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" @@ -150,7 +154,7 @@ if [ ! -f "${DOCKERFILE}" ]; then fi ### -# variables for labelling the docker image +# Variables for labelling the docker image ### BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M+00:00')" @@ -158,7 +162,7 @@ if [ -d ".git" ]; then GIT_REF="$(git rev-parse HEAD)" fi -# read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132 +# Read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132 PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}" # Get the Git information from the netbox directory @@ -169,7 +173,7 @@ if [ -d "${NETBOX_PATH}/.git" ]; then fi ### -# variables for tagging the docker image +# Variables for tagging the docker image ### DOCKER_REGISTRY="${DOCKER_REGISTRY-docker.io}" DOCKER_ORG="${DOCKER_ORG-netboxcommunity}" @@ -193,6 +197,7 @@ echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}" ### # Build each target ### +export DOCKER_BUILDKIT=${DOCKER_BUILDKIT-1} for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do echo "🏗 Building the target '${DOCKER_TARGET}'" @@ -203,6 +208,9 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do if [ "${DOCKER_TARGET}" != "main" ]; then TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}" fi + if [ -n "${GH_ACTION}" ]; then + echo "::set-env name=FINAL_DOCKER_TAG::${TARGET_DOCKER_TAG}" + fi ### # composing the additional DOCKER_SHORT_TAG, diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..534a025 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,45 @@ +version: '3.4' +services: + netbox: + image: ${IMAGE-netboxcommunity/netbox:latest} + depends_on: + - postgres + - redis + env_file: env/netbox.env + user: '101' + volumes: + - ./startup_scripts:/opt/netbox/startup_scripts:z,ro + - ./initializers:/opt/netbox/initializers:z,ro + - ./configuration:/etc/netbox/config:z,ro + - ./reports:/etc/netbox/reports:z,ro + - ./scripts:/etc/netbox/scripts:z,ro + - netbox-nginx-config:/etc/netbox-nginx:z + - netbox-static-files:/opt/netbox/netbox/static:z + - netbox-media-files:/opt/netbox/netbox/media:z + nginx: + command: nginx -c /etc/netbox-nginx/nginx.conf + image: nginx:1.17-alpine + depends_on: + - netbox + ports: + - 8080 + volumes: + - netbox-static-files:/opt/netbox/netbox/static:ro + - netbox-nginx-config:/etc/netbox-nginx/:ro + postgres: + image: postgres:11-alpine + env_file: env/postgres.env + redis: + image: redis:5-alpine + command: + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + env_file: env/redis.env +volumes: + netbox-static-files: + driver: local + netbox-nginx-config: + driver: local + netbox-media-files: + driver: local diff --git a/docker-compose.yml b/docker-compose.yml index e4961d6..10b4ad9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,8 +57,6 @@ volumes: driver: local netbox-media-files: driver: local - netbox-report-files: - driver: local netbox-postgres-data: driver: local netbox-redis-data: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 8b7481e..f555695 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,13 +1,27 @@ #!/bin/bash +# Runs on every start of the Netbox Docker container + +# Stop when an error occures set -e + +# Allows Netbox to be run as non-root users umask 002 -# wait shortly and then run db migrations (retry on error) -while ! ./manage.py migrate 2>&1; do - echo "âŗ Waiting on DB..." - sleep 3 +# Try to connect to the DB +DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3} +MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30} +CUR_DB_WAIT_TIME=0 +while ! ./manage.py migrate 2>&1 && [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do + echo "âŗ Waiting on DB... (${CUR_DB_WAIT_TIME}s / ${MAX_DB_WAIT_TIME}s)" + sleep "${DB_WAIT_TIMEOUT}" + CUR_DB_WAIT_TIME=$(( CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT )) done +if [ "${CUR_DB_WAIT_TIME}" -ge "${MAX_DB_WAIT_TIME}" ]; then + echo "❌ Waited ${MAX_DB_WAIT_TIME}s or more for the DB to become ready." + exit 1 +fi +# Create Superuser if required if [ "$SKIP_SUPERUSER" == "true" ]; then echo "↩ī¸ Skip creating the superuser" else @@ -43,18 +57,19 @@ END echo "💡 Superuser Username: ${SUPERUSER_NAME}, E-Mail: ${SUPERUSER_EMAIL}" fi +# Run the startup scripts (and initializers) if [ "$SKIP_STARTUP_SCRIPTS" == "true" ]; then echo "↩ī¸ Skipping startup scripts" else echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python fi -# copy static files +# Copy static files ./manage.py collectstatic --no-input echo "✅ Initialisation is done." -# launch whatever is passed by docker +# Launch whatever is passed by docker # (i.e. the RUN instruction in the Dockerfile) # # shellcheck disable=SC2068 diff --git a/hooks/build b/hooks/build deleted file mode 100755 index 80c4165..0000000 --- a/hooks/build +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -. hooks/common - -# shellcheck disable=SC2119 -run_build diff --git a/hooks/common b/hooks/common deleted file mode 100755 index ffa1115..0000000 --- a/hooks/common +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -ensure_jq() { - if [ ! -x "$(command -v jq)" ]; then - if [ -x "$(command -v apt-get)" ]; then - echo "🛠🛠🛠 Installing 'jq' via 'apt-get'" - apt-get update && apt-get install -y jq - else - echo "⚠ī¸âš ī¸âš ī¸ apt-get not found, unable to automatically install 'jq'." - fi - fi -} - -# Passes args to the scripts -run_build() { - echo "đŸŗđŸŗđŸŗ Building '${BUILD}' images" - case ${BUILD} in - release) - # build the latest release - # shellcheck disable=SC2068 - ./build-latest.sh $@ - ;; - prerelease) - # build the latest pre-release - # shellcheck disable=SC2068 - PRERELEASE=true ./build-latest.sh $@ - ;; - branches) - # build all branches - # shellcheck disable=SC2068 - ./build-branches.sh $@ - ;; - this) # Pull Requests - # only build the 'master' branch of netbox - # (resulting in the 'latest' docker tag) - # and the 'main' target. - DOCKER_TARGET=main ./build.sh master - ;; - *) - echo "🚨 Unrecognized build '$BUILD'." - - if [ -z "$DEBUG" ]; then - exit 1 - else - echo "⚠ī¸ Would exit here with code '1', but DEBUG is enabled." - fi - ;; - esac -} - -echo "🤖🤖🤖 Preparing build" -export DOCKER_ORG=netboxcommunity -export DOCKER_REPO=netbox -export DOCKER_REGISTRY=docker.io -# shellcheck disable=SC2153 -export BUILD="${DOCKER_TAG}" - -unset DOCKER_TAG - -ensure_jq diff --git a/hooks/push b/hooks/push deleted file mode 100755 index 6513b24..0000000 --- a/hooks/push +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -. hooks/common - -if [ "${SOURCE_BRANCH}" == "release" ] || [ "${DEBUG}" == "true" ]; then - if [ "${SOURCE_BRANCH}" != "release" ]; then - echo "⚠ī¸âš ī¸âš ī¸ Would exit, but DEBUG is '${DEBUG}'". - fi - - run_build --push-only -else - echo "⚠ī¸âš ī¸âš ī¸ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'" - exit 0 -fi diff --git a/hooks/test b/hooks/test deleted file mode 100755 index a66db12..0000000 --- a/hooks/test +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -. hooks/common - -# test on builds of 'branches' -if [ "${BUILD}" == "branches" ] \ - || [ "${DEBUG}" == "true" ]; then - ./test.sh latest - ./test.sh snapshot -# test on bulds of 'this' (i.e. pull request) -elif [ "${BUILD}" == "this" ]; then - ./test.sh latest -else - echo "đŸŗđŸŗđŸŗ No tests are implemented for build '${BUILD}'." -fi diff --git a/test.sh b/test.sh index d6d784a..627eef1 100755 --- a/test.sh +++ b/test.sh @@ -5,11 +5,14 @@ set -e # version is used by `docker-compose.yml` do determine the tag # of the Docker Image that is to be used -export VERSION=${VERSION-latest} +export IMAGE="${IMAGE-netboxcommunity/netbox:latest}" + +# The docker compose command to use +doco="docker-compose -f docker-compose.test.yml" test_netbox_unit_tests() { echo "⏱ Running Netbox Unit Tests" - docker-compose run --rm netbox ./manage.py test + $doco run --rm netbox ./manage.py test } test_initializers() { @@ -25,17 +28,20 @@ test_initializers() { mv initializers initializers_original mv initializers_test initializers - docker-compose run --rm netbox ./manage.py check + $doco run --rm netbox ./manage.py check } test_cleanup() { echo "đŸ’Ŗ Cleaning Up" - docker-compose down -v - rm -rf initializers - mv initializers_original initializers + $doco down -v + + if [ -d initializers_original ]; then + rm -rf initializers + mv initializers_original initializers + fi } -echo "đŸŗđŸŗđŸŗ Start testing '${VERSION}'" +echo "đŸŗđŸŗđŸŗ Start testing '${IMAGE}'" # Make sure the cleanup script is executed trap test_cleanup EXIT ERR @@ -43,4 +49,4 @@ trap test_cleanup EXIT ERR test_netbox_unit_tests test_initializers -echo "đŸŗđŸŗđŸŗ Done testing '${VERSION}'" +echo "đŸŗđŸŗđŸŗ Done testing '${IMAGE}'" From 57afeec94f9e0a05d95d8af80ecb4902a899f98d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sun, 29 Dec 2019 22:09:32 +0100 Subject: [PATCH 23/40] Netbox is not yet compatible with Python 3.9 --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3290e27..4a67b5e 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,7 +15,7 @@ jobs: - '' # use the default of the DOCKERFILE - python:3.7-alpine - python:3.8-alpine - - python:3.9-rc-alpine + # - python:3.9-rc-alpine # Netbox does not work with Python 3.9 yet. fail-fast: false runs-on: ubuntu-latest name: Builds new Netbox Docker Images From 3b56c827f95bd5cda2a3895f0521c9fd97942f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 17:21:47 +0100 Subject: [PATCH 24/40] Improve test.sh Don't move the original initializers directory, but rather use the one with the modified initializers in it. --- .gitignore | 1 + docker-compose.test.yml | 2 +- test.sh | 39 +++++++++++++++++++++++---------------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 4b029d7..97aa1b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.sql.gz .netbox +.initializers docker-compose.override.yml diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 534a025..9420617 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -9,7 +9,7 @@ services: user: '101' volumes: - ./startup_scripts:/opt/netbox/startup_scripts:z,ro - - ./initializers:/opt/netbox/initializers:z,ro + - ./${INITIALIZERS_DIR-initializers}:/opt/netbox/initializers:z,ro - ./configuration:/etc/netbox/config:z,ro - ./reports:/etc/netbox/reports:z,ro - ./scripts:/etc/netbox/scripts:z,ro diff --git a/test.sh b/test.sh index 627eef1..2b74847 100755 --- a/test.sh +++ b/test.sh @@ -10,24 +10,31 @@ export IMAGE="${IMAGE-netboxcommunity/netbox:latest}" # The docker compose command to use doco="docker-compose -f docker-compose.test.yml" +INITIALIZERS_DIR=".initializers" + +test_setup() { + echo "🏗 Setup up test environment" + if [ -d "${INITIALIZERS_DIR}" ]; then + rm -rf "${INITIALIZERS_DIR}" + fi + + mkdir "${INITIALIZERS_DIR}" + ( + cd initializers + for script in *.yml; do + sed -E 's/^# //' "${script}" > "../${INITIALIZERS_DIR}/${script}" + done + ) +} + test_netbox_unit_tests() { echo "⏱ Running Netbox Unit Tests" $doco run --rm netbox ./manage.py test } test_initializers() { - echo "🏗 Testing Initializers" - - mkdir initializers_test - ( - cd initializers - for script in *.yml; do - sed -E 's/^# //' "${script}" > "../initializers_test/${script}" - done - ) - mv initializers initializers_original - mv initializers_test initializers - + echo "🏭 Testing Initializers" + export INITIALIZERS_DIR $doco run --rm netbox ./manage.py check } @@ -35,9 +42,8 @@ test_cleanup() { echo "đŸ’Ŗ Cleaning Up" $doco down -v - if [ -d initializers_original ]; then - rm -rf initializers - mv initializers_original initializers + if [ -d "${INITIALIZERS_DIR}" ]; then + rm -rf "${INITIALIZERS_DIR}" fi } @@ -45,8 +51,9 @@ echo "đŸŗđŸŗđŸŗ Start testing '${IMAGE}'" # Make sure the cleanup script is executed trap test_cleanup EXIT ERR +test_setup -test_netbox_unit_tests +# test_netbox_unit_tests test_initializers echo "đŸŗđŸŗđŸŗ Done testing '${IMAGE}'" From cfbd037f79e58fbd881cc99dacc4adeba9f1d3d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 15:21:53 +0100 Subject: [PATCH 25/40] Enable tests with Python 3.9 --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4a67b5e..3290e27 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,7 +15,7 @@ jobs: - '' # use the default of the DOCKERFILE - python:3.7-alpine - python:3.8-alpine - # - python:3.9-rc-alpine # Netbox does not work with Python 3.9 yet. + - python:3.9-rc-alpine fail-fast: false runs-on: ubuntu-latest name: Builds new Netbox Docker Images From dd0aee081a8e0f818fe8a95a2efd44449838ee5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 18:10:36 +0100 Subject: [PATCH 26/40] Fix failing GH Action --- .github/workflows/push.yml | 10 +++++++--- .github/workflows/release.yml | 22 ++++++++++++++++------ build-latest.sh | 4 ++++ build.sh | 1 + test.sh | 10 ++++++++++ 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3290e27..8f9ca18 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -20,12 +20,16 @@ jobs: runs-on: ubuntu-latest name: Builds new Netbox Docker Images steps: - - name: Checkout + - id: git-checkout + name: Checkout uses: actions/checkout@v1 - - name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}' + - id: docker-build + name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}' run: ${{ matrix.build_cmd }} env: DOCKER_FROM: ${{ matrix.docker_from }} GH_ACTION: enable - - name: Test the image + - id: docker-test + name: Test the image run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh + if: steps.docker-build.outputs.skipped != 'true' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 72a0bdb..9a00d3d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,15 +17,20 @@ jobs: runs-on: ubuntu-latest name: Builds new Netbox Docker Images steps: - - name: Checkout + - id: git-checkout + name: Checkout uses: actions/checkout@v1 - - name: Build the image with '${{ matrix.build_cmd }}' + - id: docker-build + name: Build the image with '${{ matrix.build_cmd }}' run: ${{ matrix.build_cmd }} env: GH_ACTION: enable - - name: Test the image + - id: docker-test + name: Test the image run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh - - name: Login to the Docker Registry + if: steps.docker-build.outputs.skipped != 'true' + - id: registry-login + name: Login to the Docker Registry run: | echo "::add-mask::$DOCKERHUB_USERNAME" echo "::add-mask::$DOCKERHUB_PASSWORD" @@ -33,7 +38,12 @@ jobs: env: DOCKERHUB_USERNAME: ${{ secrets.dockerhub_username }} DOCKERHUB_PASSWORD: ${{ secrets.dockerhub_password }} - - name: Push the image + if: steps.docker-build.outputs.skipped != 'true' + - id: registry-push + name: Push the image run: ${{ matrix.build_cmd }} --push-only - - name: Logout of the Docker Registry + if: steps.docker-build.outputs.skipped != 'true' + - id: registry-logout + name: Logout of the Docker Registry run: docker logout "${DOCKER_REGISTRY}" + if: steps.docker-build.outputs.skipped != 'true' diff --git a/build-latest.sh b/build-latest.sh index 31a0f76..468dffe 100755 --- a/build-latest.sh +++ b/build-latest.sh @@ -66,6 +66,10 @@ if [ "${PRERELEASE}" == "true" ]; then echo "❎ Latest unstable version '${VERSION}' is not higher than the latest stable version '$STABLE_VERSION'." if [ -z "$DEBUG" ]; then + if [ -n "${GH_ACTION}" ]; then + echo "::set-output name=skipped::true" + fi + exit 0 else echo "⚠ī¸ Would exit here with code '0', but DEBUG is enabled." diff --git a/build.sh b/build.sh index 94cbe6a..82a6e34 100755 --- a/build.sh +++ b/build.sh @@ -210,6 +210,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do fi if [ -n "${GH_ACTION}" ]; then echo "::set-env name=FINAL_DOCKER_TAG::${TARGET_DOCKER_TAG}" + echo "::set-output name=skipped::false" fi ### diff --git a/test.sh b/test.sh index 627eef1..419589b 100755 --- a/test.sh +++ b/test.sh @@ -7,6 +7,16 @@ set -e # of the Docker Image that is to be used export IMAGE="${IMAGE-netboxcommunity/netbox:latest}" +if [ -z "${IMAGE}" ]; then + echo "⚠ī¸ No image defined" + + if [ -z "${DEBUG}" ]; then + exit 1; + else + echo "⚠ī¸ Would 'exit 1' here, but DEBUG is '${DEBUG}'." + fi +fi + # The docker compose command to use doco="docker-compose -f docker-compose.test.yml" From 25671d42a50c884d0093c0499a632d17e4a9875b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 20 Jan 2020 08:37:18 +0100 Subject: [PATCH 27/40] Re-enable Netbox unit tests --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index 2b74847..6684f7c 100755 --- a/test.sh +++ b/test.sh @@ -53,7 +53,7 @@ echo "đŸŗđŸŗđŸŗ Start testing '${IMAGE}'" trap test_cleanup EXIT ERR test_setup -# test_netbox_unit_tests +test_netbox_unit_tests test_initializers echo "đŸŗđŸŗđŸŗ Done testing '${IMAGE}'" From 355f9d4cf7e5fb7b2df412da412da139c3b579a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 16 Dec 2019 12:51:59 +0100 Subject: [PATCH 28/40] Prepare for Netbox 2.7 --- Dockerfile | 4 +++- configuration/configuration.py | 31 ++++++++++++++++++------------- docker-compose.yml | 8 ++++++++ env/netbox.env | 5 ++++- env/redis-cache.env | 1 + 5 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 env/redis-cache.env diff --git a/Dockerfile b/Dockerfile index 82576c1..7764248 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,9 @@ RUN pip install --prefix="/install" --no-warn-script-location \ # ruamel is used in startup_scripts 'ruamel.yaml>=0.15,<0.16' \ # django_auth_ldap is required for ldap - django_auth_ldap + django_auth_ldap \ +# django-storages was introduced in 2.7 and is optional + django-storages ARG NETBOX_PATH COPY ${NETBOX_PATH}/requirements.txt / diff --git a/configuration/configuration.py b/configuration/configuration.py index 4199b9d..af121d9 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -37,9 +37,9 @@ DATABASE = { # PostgreSQL password 'HOST': os.environ.get('DB_HOST', 'localhost'), # Database server 'PORT': os.environ.get('DB_PORT', ''), # Database port (leave blank for default) - 'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')}, + 'OPTIONS': {'sslmode': os.environ.get('DB_SSLMODE', 'prefer')}, # Database connection SSLMODE - 'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')), + 'CONN_MAX_AGE': int(os.environ.get('DB_CONN_MAX_AGE', '300')), # Database connection persistence } @@ -51,13 +51,22 @@ SECRET_KEY = os.environ.get('SECRET_KEY', read_secret('secret_key')) # Redis database settings. The Redis database is used for caching and background processing such as webhooks REDIS = { - 'HOST': os.environ.get('REDIS_HOST', 'localhost'), - 'PORT': int(os.environ.get('REDIS_PORT', 6379)), - 'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')), - 'DATABASE': os.environ.get('REDIS_DATABASE', '0'), - 'CACHE_DATABASE': os.environ.get('REDIS_CACHE_DATABASE', '1'), - 'DEFAULT_TIMEOUT': os.environ.get('REDIS_TIMEOUT', '300'), - 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', + 'webhooks': { + 'HOST': os.environ.get('REDIS_HOST', 'localhost'), + 'PORT': int(os.environ.get('REDIS_PORT', 6379)), + 'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')), + 'DATABASE': int(os.environ.get('REDIS_DATABASE', 0)), + 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_TIMEOUT', 300)), + 'SSL': os.environ.get('REDIS_SSL', 'False').lower() == 'true', + }, + 'caching': { + 'HOST': os.environ.get('REDIS_CACHE_HOST', os.environ.get('REDIS_HOST', 'localhost')), + 'PORT': int(os.environ.get('REDIS_CACHE_PORT', os.environ.get('REDIS_PORT', 6379))), + 'PASSWORD': os.environ.get('REDIS_CACHE_PASSWORD', os.environ.get('REDIS_PASSWORD', read_secret('redis_cache_password'))), + 'DATABASE': int(os.environ.get('REDIS_CACHE_DATABASE', 1)), + 'DEFAULT_TIMEOUT': int(os.environ.get('REDIS_CACHE_TIMEOUT', os.environ.get('REDIS_TIMEOUT', 300))), + 'SSL': os.environ.get('REDIS_CACHE_SSL', os.environ.get('REDIS_SSL', 'False')).lower() == 'true', + }, } ######################### @@ -172,10 +181,6 @@ SCRIPTS_ROOT = os.environ.get('SCRIPTS_ROOT', '/etc/netbox/scripts') # Time zone (default: UTC) TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC') -# The Webhook event backend is disabled by default. Set this to True to enable it. Note that this requires a Redis -# database be configured and accessible by NetBox (see `REDIS` below). -WEBHOOKS_ENABLED = os.environ.get('WEBHOOKS_ENABLED', 'False').lower() == 'true' - # Date/time formatting. See the following link for supported formats: # https://docs.djangoproject.com/en/dev/ref/templates/builtins/#date DATE_FORMAT = os.environ.get('DATE_FORMAT', 'N j, Y') diff --git a/docker-compose.yml b/docker-compose.yml index 10b4ad9..3806480 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ services: depends_on: - postgres - redis + - redis-cache - netbox-worker env_file: env/netbox.env user: '101' @@ -50,6 +51,13 @@ services: env_file: env/redis.env volumes: - netbox-redis-data:/data + redis-cache: + image: redis:5-alpine + command: + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + env_file: env/redis.env volumes: netbox-static-files: driver: local diff --git a/env/netbox.env b/env/netbox.env index 4a75905..aaa7482 100644 --- a/env/netbox.env +++ b/env/netbox.env @@ -17,8 +17,11 @@ MAX_PAGE_SIZE=1000 REDIS_HOST=redis REDIS_PASSWORD=H733Kdjndks81 REDIS_DATABASE=0 -REDIS_CACHE_DATABASE=1 REDIS_SSL=false +REDIS_CACHE_HOST=redis-cache +REDIS_CACHE_PASSWORD=t4Ph722qJ5QHeQ1qfu36 +REDIS_CACHE_DATABASE=0 +REDIS_CACHE_SSL=false SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj SKIP_STARTUP_SCRIPTS=false SKIP_SUPERUSER=false diff --git a/env/redis-cache.env b/env/redis-cache.env new file mode 100644 index 0000000..6285c33 --- /dev/null +++ b/env/redis-cache.env @@ -0,0 +1 @@ +REDIS_PASSWORD=t4Ph722qJ5QHeQ1qfu36 From c001626b85ba07bf2203faee16b09372fbe42ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 15:22:16 +0100 Subject: [PATCH 29/40] Update Custom Fields Initializer for Netbox 2.7 The custom field database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your custom_fields.yml file as follows: - type must be lowercase - the `selection` type was changed to `select` - the filter_logic must be lower case This is to achieve compatibility with the naming schema that Netbox uses internally. It allows us to become forward compatible in case Netbox ever introduces a new type for custom fields. See the diff of this commit for further information how this is meant. --- initializers/custom_fields.yml | 23 +++++++++++++++++++---- startup_scripts/020_custom_fields.py | 18 +----------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/initializers/custom_fields.yml b/initializers/custom_fields.yml index 0b6472a..4085ab0 100644 --- a/initializers/custom_fields.yml +++ b/initializers/custom_fields.yml @@ -1,3 +1,18 @@ +## Possible Choices: +## type: +## - text +## - integer +## - boolean +## - date +## - url +## - select +## filter_logic: +## - disabled +## - loose +## - exact +## +## Examples: + # text_field: # type: text # label: Custom Text @@ -22,8 +37,8 @@ # weight: 10 # on_objects: # - tenancy.models.Tenant -# selection_field: -# type: selection +# select_field: +# type: select # label: Choose between items # required: false # filter_logic: exact @@ -41,8 +56,8 @@ # weight: 50 # - value: Fourth Item # weight: 40 -# selection_field_auto_weight: -# type: selection +# select_field_auto_weight: +# type: select # label: Choose between items # required: false # filter_logic: loose diff --git a/startup_scripts/020_custom_fields.py b/startup_scripts/020_custom_fields.py index 76a32bb..2f6ba72 100644 --- a/startup_scripts/020_custom_fields.py +++ b/startup_scripts/020_custom_fields.py @@ -1,19 +1,9 @@ -from extras.constants import CF_TYPE_TEXT, CF_TYPE_INTEGER, CF_TYPE_BOOLEAN, CF_TYPE_DATE, CF_TYPE_URL, CF_TYPE_SELECT, CF_FILTER_CHOICES from extras.models import CustomField, CustomFieldChoice from ruamel.yaml import YAML from pathlib import Path import sys -text_to_fields = { - 'boolean': CF_TYPE_BOOLEAN, - 'date': CF_TYPE_DATE, - 'integer': CF_TYPE_INTEGER, - 'selection': CF_TYPE_SELECT, - 'text': CF_TYPE_TEXT, - 'url': CF_TYPE_URL, -} - def get_class_for_class_path(class_path): import importlib from django.contrib.contenttypes.models import ContentType @@ -42,12 +32,6 @@ with file.open('r') as stream: if cf_details.get('description', 0): custom_field.description = cf_details['description'] - # If no filter_logic is specified then it will default to 'Loose' - if cf_details.get('filter_logic', 0): - for choice_id, choice_text in CF_FILTER_CHOICES: - if choice_text.lower() == cf_details['filter_logic']: - custom_field.filter_logic = choice_id - if cf_details.get('label', 0): custom_field.label = cf_details['label'] @@ -58,7 +42,7 @@ with file.open('r') as stream: custom_field.required = cf_details['required'] if cf_details.get('type', 0): - custom_field.type = text_to_fields[cf_details['type']] + custom_field.type = cf_details['type'] if cf_details.get('weight', 0): custom_field.weight = cf_details['weight'] From 7b914d31d66f56bb9cf6ab8fbba131a23a6127a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 15:44:19 +0100 Subject: [PATCH 30/40] Update Rack Initializer for Netbox 2.7 The rack database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your racks.yml file as follows: - Rack types must match one of the 5 rack types given, e.g. '4-post-cabinet'. - Rack width must match one of the 2 rack widths given, i.e. '19' or '23'. See the diff of this commit for further information how this is meant. --- initializers/racks.yml | 28 ++++++++++++++++++++++------ startup_scripts/080_racks.py | 9 --------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/initializers/racks.yml b/initializers/racks.yml index 3a31526..51502de 100644 --- a/initializers/racks.yml +++ b/initializers/racks.yml @@ -1,16 +1,32 @@ +## Possible Choices: +## width: +## - 19 +## - 23 +## types: +## - 2-post-frame +## - 4-post-frame +## - 4-post-cabinet +## - wall-frame +## - wall-cabinet +## outer_unit: +## - mm +## - in +## +## Examples: + # - site: AMS 1 # name: rack-01 # role: Role 1 -# type: 4-post cabinet -# width: 19 inches +# type: 4-post-cabinet +# width: 19 # u_height: 47 # custom_fields: # text_field: Description # - site: AMS 2 # name: rack-02 # role: Role 2 -# type: 4-post cabinet -# width: 19 inches +# type: 4-post-cabinet +# width: 19 # u_height: 47 # custom_fields: # text_field: Description @@ -18,8 +34,8 @@ # name: rack-03 # group: cage 101 # role: Role 3 -# type: 4-post cabinet -# width: 19 inches +# type: 4-post-cabinet +# width: 19 # u_height: 47 # custom_fields: # text_field: Description diff --git a/startup_scripts/080_racks.py b/startup_scripts/080_racks.py index 05bca10..ed7713d 100644 --- a/startup_scripts/080_racks.py +++ b/startup_scripts/080_racks.py @@ -1,7 +1,6 @@ from dcim.models import Site, RackRole, Rack, RackGroup from tenancy.models import Tenant from extras.models import CustomField, CustomFieldValue -from dcim.constants import RACK_TYPE_CHOICES, RACK_WIDTH_CHOICES from ruamel.yaml import YAML from pathlib import Path import sys @@ -41,14 +40,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - for rack_type in RACK_TYPE_CHOICES: - if params['type'] in rack_type: - params['type'] = rack_type[0] - - for rack_width in RACK_WIDTH_CHOICES: - if params['width'] in rack_width: - params['width'] = rack_width[0] - rack, created = Rack.objects.get_or_create(**params) if created: From 4a58676647e41115d5ecc10c4e9035ca73db53ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 15:52:17 +0100 Subject: [PATCH 31/40] Update Device Initializer for Netbox 2.7 The device database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your devices.yml file as follows: - Make sure the rack face is spelled lowercase. See the diff of this commit for further information how this is meant. --- initializers/devices.yml | 21 ++++++++++++++++++--- startup_scripts/130_devices.py | 7 ------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/initializers/devices.yml b/initializers/devices.yml index 0beb6f2..708b68c 100644 --- a/initializers/devices.yml +++ b/initializers/devices.yml @@ -1,9 +1,24 @@ +## Possible Choices: +## face: +## - front +## - rear +## status: +## - offline +## - active +## - planned +## - staged +## - failed +## - inventory +## - decommissioning +## +## Examples: + # - name: server01 # device_role: server # device_type: Other # site: AMS 1 # rack: rack-01 -# face: Front +# face: front # position: 1 # custom_fields: # text_field: Description @@ -12,7 +27,7 @@ # device_type: Other # site: AMS 2 # rack: rack-02 -# face: Front +# face: front # position: 2 # custom_fields: # text_field: Description @@ -21,7 +36,7 @@ # device_type: Other # site: SING 1 # rack: rack-03 -# face: Front +# face: front # position: 3 # custom_fields: # text_field: Description diff --git a/startup_scripts/130_devices.py b/startup_scripts/130_devices.py index 2d8d3ca..4217549 100644 --- a/startup_scripts/130_devices.py +++ b/startup_scripts/130_devices.py @@ -1,5 +1,4 @@ from dcim.models import Site, Rack, DeviceRole, DeviceType, Device, Platform -from dcim.constants import RACK_FACE_CHOICES from ipam.models import IPAddress from virtualization.models import Cluster from tenancy.models import Tenant @@ -49,12 +48,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'face' in params: - for rack_face in RACK_FACE_CHOICES: - if params['face'] in rack_face: - params['face'] = rack_face[0] - break - device, created = Device.objects.get_or_create(**params) if created: From 8d8b9a157e219ce3ed5944f8fd19ca668b4c6cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 15:59:21 +0100 Subject: [PATCH 32/40] Update VLAN Initializer for Netbox 2.7 The VLAN database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your vlans.yml file as follows: - Make sure the status is spelled lowercase. See the diff of this commit for further information how this is meant. --- initializers/vlans.yml | 12 ++++++++++-- startup_scripts/210_vlans.py | 7 ------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/initializers/vlans.yml b/initializers/vlans.yml index 26532c7..a8cd521 100644 --- a/initializers/vlans.yml +++ b/initializers/vlans.yml @@ -1,11 +1,19 @@ +## Possible Choices: +## status: +## - active +## - reserved +## - deprecated +## +## Examples: + # - name: vlan1 # site: AMS 1 -# status: Active +# status: active # vid: 5 # role: Main Management # description: VLAN 5 for MGMT # - group: VLAN group 2 # name: vlan2 # site: AMS 1 -# status: Active +# status: active # vid: 1300 diff --git a/startup_scripts/210_vlans.py b/startup_scripts/210_vlans.py index e3a0ef1..ab6bd2b 100644 --- a/startup_scripts/210_vlans.py +++ b/startup_scripts/210_vlans.py @@ -1,6 +1,5 @@ from dcim.models import Site from ipam.models import VLAN, VLANGroup, Role -from ipam.constants import VLAN_STATUS_CHOICES from tenancy.models import Tenant, TenantGroup from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -35,12 +34,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'status' in params: - for vlan_status in VLAN_STATUS_CHOICES: - if params['status'] in vlan_status: - params['status'] = vlan_status[0] - break - vlan, created = VLAN.objects.get_or_create(**params) if created: From f3403cd0f5a5c26e5aea9afed2e4d3a5a129a1b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 16:03:45 +0100 Subject: [PATCH 33/40] Update VM Initializer for Netbox 2.7 The vm database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your virtual_machines.yml file as follows: - Make sure the status is spelled lowercase. See the diff of this commit for further information how this is meant. --- initializers/prefixes.yml | 15 ++++++++++++--- initializers/virtual_machines.yml | 12 ++++++++++-- startup_scripts/220_prefixes.py | 7 ------- startup_scripts/230_virtual_machines.py | 7 ------- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/initializers/prefixes.yml b/initializers/prefixes.yml index a7e6815..fbf3eee 100644 --- a/initializers/prefixes.yml +++ b/initializers/prefixes.yml @@ -1,13 +1,22 @@ +## Possible Choices: +## status: +## - container +## - active +## - reserved +## - deprecated +## +## Examples: + # - description: prefix1 # prefix: 10.1.1.0/24 # site: AMS 1 -# status: Active +# status: active # tenant: tenant1 # vlan: vlan1 # - description: prefix2 # prefix: 10.1.2.0/24 # site: AMS 2 -# status: Active +# status: active # tenant: tenant2 # vlan: vlan2 # is_pool: true @@ -15,6 +24,6 @@ # - description: ipv6 prefix1 # prefix: 2001:db8:a000:1::/64 # site: AMS 2 -# status: Active +# status: active # tenant: tenant2 # vlan: vlan2 diff --git a/initializers/virtual_machines.yml b/initializers/virtual_machines.yml index 5da75a2..2122920 100644 --- a/initializers/virtual_machines.yml +++ b/initializers/virtual_machines.yml @@ -1,10 +1,18 @@ +## Possible Choices: +## status: +## - active +## - offline +## - staged +## +## Examples: + # - cluster: cluster1 # comments: VM1 # disk: 200 # memory: 4096 # name: virtual machine 1 # platform: Platform 2 -# status: Active +# status: active # tenant: tenant1 # vcpus: 8 # - cluster: cluster1 @@ -13,6 +21,6 @@ # memory: 2048 # name: virtual machine 2 # platform: Platform 2 -# status: Active +# status: active # tenant: tenant1 # vcpus: 8 diff --git a/startup_scripts/220_prefixes.py b/startup_scripts/220_prefixes.py index a832c88..d13578a 100644 --- a/startup_scripts/220_prefixes.py +++ b/startup_scripts/220_prefixes.py @@ -1,6 +1,5 @@ from dcim.models import Site from ipam.models import Prefix, VLAN, Role, VRF -from ipam.constants import PREFIX_STATUS_CHOICES from tenancy.models import Tenant, TenantGroup from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -38,12 +37,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'status' in params: - for prefix_status in PREFIX_STATUS_CHOICES: - if params['status'] in prefix_status: - params['status'] = prefix_status[0] - break - prefix, created = Prefix.objects.get_or_create(**params) if created: diff --git a/startup_scripts/230_virtual_machines.py b/startup_scripts/230_virtual_machines.py index 065b600..449df8a 100644 --- a/startup_scripts/230_virtual_machines.py +++ b/startup_scripts/230_virtual_machines.py @@ -1,6 +1,5 @@ from dcim.models import Site, Platform, DeviceRole from virtualization.models import Cluster, VirtualMachine -from virtualization.constants import VM_STATUS_CHOICES from tenancy.models import Tenant from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -43,12 +42,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'status' in params: - for vm_status in VM_STATUS_CHOICES: - if params['status'] in vm_status: - params['status'] = vm_status[0] - break - virtual_machine, created = VirtualMachine.objects.get_or_create(**params) if created: From 74a0e2cf6ee38d28d312386514a9e3ce8116b015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 16:14:09 +0100 Subject: [PATCH 34/40] Update Interfaces Initializer for Netbox 2.7 The interface database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your dcim_interfaces.yml file as follows: - Make sure the type is a value out of the possible choices. See the diff of this commit for further information how this is meant. --- initializers/dcim_interfaces.yml | 14 ++++++++++++-- startup_scripts/250_dcim_interfaces.py | 11 ----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/initializers/dcim_interfaces.yml b/initializers/dcim_interfaces.yml index 18920fe..4030530 100644 --- a/initializers/dcim_interfaces.yml +++ b/initializers/dcim_interfaces.yml @@ -1,8 +1,18 @@ +## Possible Choices: +## type: +## - virtual +## - lag +## - 1000base-t +## - ... and many more. See for yourself: +## https://github.com/netbox-community/netbox/blob/295d4f0394b431351c0cb2c3ecc791df68c6c2fb/netbox/dcim/choices.py#L510 +## +## Examples: + # - device: server01 # enabled: true -# type: Virtual +# type: virtual # name: to-server02 # - device: server02 # enabled: true -# type: Virtual +# type: virtual # name: to-server01 diff --git a/startup_scripts/250_dcim_interfaces.py b/startup_scripts/250_dcim_interfaces.py index ce7e7bd..ec30b5c 100644 --- a/startup_scripts/250_dcim_interfaces.py +++ b/startup_scripts/250_dcim_interfaces.py @@ -1,5 +1,4 @@ from dcim.models import Interface, Device -from dcim.constants import IFACE_TYPE_CHOICES from extras.models import CustomField, CustomFieldValue from ruamel.yaml import YAML @@ -28,16 +27,6 @@ with file.open('r') as stream: params[assoc] = model.objects.get(**query) - if 'type' in params: - for outer_list in IFACE_TYPE_CHOICES: - for type_choices in outer_list[1]: - if params['type'] in type_choices: - params['type'] = type_choices[0] - break - else: - continue - break - interface, created = Interface.objects.get_or_create(**params) if created: From 27f671e41ad561ad3ecf3493e410624c2c503c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Fri, 17 Jan 2020 16:18:52 +0100 Subject: [PATCH 35/40] Update IP Initializer for Netbox 2.7 The ip address database model has changed in Netbox 2.7. Therefore the initializer code, that was used before, broke. As a user, you will need to update your ip_addresses.yml file as follows: - Make sure the status is written in lower case. See the diff of this commit for further information how this is meant. --- initializers/ip_addresses.yml | 30 +++++++++++++++++++++++------ startup_scripts/260_ip_addresses.py | 7 ------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/initializers/ip_addresses.yml b/initializers/ip_addresses.yml index 5738749..6ac38e9 100644 --- a/initializers/ip_addresses.yml +++ b/initializers/ip_addresses.yml @@ -1,26 +1,44 @@ +## Possible Choices: +## status: +## - active +## - reserved +## - deprecated +## - dhcp +## role: +## - loopback +## - secondary +## - anycast +## - vip +## - vrrp +## - hsrp +## - glbp +## - carp +## +## Examples: + # - address: 10.1.1.1/24 # device: server01 # interface: to-server02 -# status: Active +# status: active # vrf: vrf1 # - address: 2001:db8:a000:1::1/64 # device: server01 # interface: to-server02 -# status: Active +# status: active # vrf: vrf1 # - address: 10.1.1.2/24 # device: server02 # interface: to-server01 -# status: Active +# status: active # - address: 2001:db8:a000:1::2/64 # device: server02 # interface: to-server01 -# status: Active +# status: active # - address: 10.1.1.10/24 # description: reserved IP -# status: Reserved +# status: reserved # tenant: tenant1 # - address: 2001:db8:a000:1::10/64 # description: reserved IP -# status: Reserved +# status: reserved # tenant: tenant1 diff --git a/startup_scripts/260_ip_addresses.py b/startup_scripts/260_ip_addresses.py index f7f2bc7..d109a36 100644 --- a/startup_scripts/260_ip_addresses.py +++ b/startup_scripts/260_ip_addresses.py @@ -1,5 +1,4 @@ from ipam.models import IPAddress, VRF -from ipam.constants import IPADDRESS_STATUS_CHOICES from dcim.models import Device, Interface from virtualization.models import VirtualMachine from tenancy.models import Tenant @@ -49,12 +48,6 @@ with file.open('r') as stream: query = { field: params.pop(assoc) } params[assoc] = model.objects.get(**query) - if 'status' in params: - for ip_status in IPADDRESS_STATUS_CHOICES: - if params['status'] in ip_status: - params['status'] = ip_status[0] - break - ip_address, created = IPAddress.objects.get_or_create(**params) if created: From 2b21e14c2c69d1c747d1a349c365521d2f877054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 13 Jan 2020 18:14:23 +0100 Subject: [PATCH 36/40] Check that `develop` is target in PR This is an update to the PR template. It mostly adds text and some checks related to how good the template is filled in and whether the `develop` branch was selected as base. --- .github/pull_request_template.md | 46 +++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1c3885f..792055a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,13 +1,31 @@ + + -But if there is already a related issue, please put it's number here. E.g. #123 or N/A --> Related Issue: ## New Behavior ... @@ -15,7 +33,8 @@ please describe in a few words the behavior your PR implements ## Contrast to Current Behavior ... @@ -24,12 +43,15 @@ please describe in a few words how the new behavior is different from the curren ... @@ -38,7 +60,17 @@ Please make your case here: ... + +## Double Check + + + +* [ ] I have read the comments and followed the PR template. +* [ ] I have provided a explained my PR according to the information in the comments. +* [ ] My PR targets the `develop` branch. From 1bad9b4fa89639e205472f49d5a0e8f7e41be8fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Mon, 13 Jan 2020 18:24:51 +0100 Subject: [PATCH 37/40] Fix typo in pull request template --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 792055a..6f755c3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -72,5 +72,5 @@ Please put an x into the brackets (like `[x]`) if you've completed that task. --> * [ ] I have read the comments and followed the PR template. -* [ ] I have provided a explained my PR according to the information in the comments. +* [ ] I have provided and explained my PR according to the information in the comments. * [ ] My PR targets the `develop` branch. From 8892ea99369ae6115d47588886373179b51232a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Sat, 18 Jan 2020 11:48:08 +0100 Subject: [PATCH 38/40] Ask for a release note entry --- .github/pull_request_template.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6f755c3..32b34f0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -65,6 +65,15 @@ If the Wiki must be updated, please make a suggestion below. ... +## Proposed Release Note Entry + + + +... + ## Double Check