Merge pull request #91 from ninech/webhooks-backend

Webhooks Backend (based on #90)

Dockerfile

@@ -6,13 +6,13 @@ RUN apk add --no-cache \
       ca-certificates \
       cyrus-sasl-dev \
       graphviz \
-      ttf-ubuntu-font-family \
       jpeg-dev \
       libffi-dev \
       libxml2-dev \
       libxslt-dev \
       openldap-dev \
       postgresql-dev \
+      ttf-ubuntu-font-family \
       wget
 
 RUN pip install \
@@ -21,7 +21,13 @@ RUN pip install \
 # napalm is used for gathering information from network devices
       napalm \
 # ruamel is used in startup_scripts
-      ruamel.yaml
+      ruamel.yaml \
+# pinning django to the version required by netbox
+# adding it here, to install the correct version of
+# django-rq
+      'Django>=1.11,<2.1' \
+# django-rq is used for webhooks
+      django-rq
 
 WORKDIR /opt
 
@@ -45,8 +51,6 @@ WORKDIR /opt/netbox/netbox
 
 ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
 
-VOLUME ["/etc/netbox-nginx/"]
-
 CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"]
 
 LABEL SRC_URL="$URL"

README.md

@@ -54,7 +54,8 @@ To ensure this, compare the output of `docker --version` and `docker-compose --v
 
 ## Configuration
 
-You can configure the app using environment variables. These are defined in `netbox.env`.
+You can configure the app using environment variables.
+These are defined in `netbox.env`.
 Read [Environment Variables in Compose][compose-env] to understand about the various possibilities to overwrite these variables.
 (The easiest solution being simply adjusting that file.)
 
@@ -70,11 +71,12 @@ For example defining `ALLOWED_HOSTS=localhost ::1 127.0.0.1` would allows access
 The default settings are optimized for (local) development environments.
 You should therefore adjust the configuration for production setups, at least the following variables:
 
-* `ALLOWED_HOSTS`: Add all URLs that lead to your NetBox instance.
+* `ALLOWED_HOSTS`: Add all URLs that lead to your NetBox instance, space separated. E.g. `ALLOWED_HOSTS=netbox.mycorp.com server042.mycorp.com 2a02:123::42 10.0.0.42 localhost ::1 127.0.0.1` (It's good advice to always allow localhost connections for easy debugging, i.e. `localhost ::1 127.0.0.1`.)
-* `DB_*`: Use a persistent database.
+* `DB_*`: Use your own persistent database. Don't use the default passwords!
 * `EMAIL_*`: Use your own mailserver.
 * `MAX_PAGE_SIZE`: Use the recommended default of 1000.
-* `SUPERUSER_*`: Only define those variables during the initial setup, and drop them once the DB is set up.
+* `SUPERUSER_*`: Only define those variables during the initial setup, and drop them once the DB is set up. Don't use the default passwords!
+* `REDIS_*`: Use your own persistent redis. Don't use the default passwords!
 
 ### Running on Docker Swarm / Kubernetes / OpenShift
 
@@ -95,6 +97,7 @@ If a secret is defined by an environment variable and in the respective file at
 * `SECRET_KEY`: `/run/secrets/secret_key`
 * `EMAIL_PASSWORD`: `/run/secrets/email_password`
 * `NAPALM_PASSWORD`: `/run/secrets/napalm_password`
+* `REDIS_PASSWORD`: `/run/secrets/redis_password`
 
 Please also consider [the advice about running NetBox in production](#production) above!
 
@@ -249,6 +252,32 @@ If your issue is not here, look through [the existing issues][issues] and eventu
 * To create a database backup run `docker-compose exec postgres sh -c 'pg_dump -cU $POSTGRES_USER $POSTGRES_DB' | gzip > db_dump.sql.gz`
 * To restore that database backup run `gunzip -c db_dump.sql.gz | docker exec -i $(docker-compose ps -q postgres) sh -c 'psql -U $POSTGRES_USER $POSTGRES_DB'`.
 
+### Nginx doesn't start
+
+As a first step, stop your docker-compose setup.
+Then locate the `netbox-nginx-config` volume and remove it:
+
+```bash
+# Stop your local netbox-docker installation
+$ docker-compose down
+
+# Find the volume
+$ docker volume ls | grep netbox-nginx-config
+local               netbox-docker_netbox-nginx-config
+
+# Remove the volume
+$ docker volume rm netbox-docker_netbox-nginx-config
+netbox-docker_netbox-nginx-config
+```
+
+Now start everything up again.
+
+If this didn't help, try to see if there's anything in the logs indicating why nginx doesn't start:
+
+```bash
+$ docker-compose logs -f nginx
+```
+
 ### Getting a "Bad Request (400)"
 
 > When connecting to the NetBox instance, I get a "Bad Request (400)" error.
@@ -257,26 +286,77 @@ This usually happens when the `ALLOWED_HOSTS` variable is not set correctly.
 
 ### How to upgrade
 
-> How do I update to a newer version?
+> How do I update to a newer version of netbox?
 
 It should be sufficient to pull the latest image from Docker Hub, stopping the container and starting it up again:
 
 ```bash
 docker-compose pull netbox
-docker-compose stop netbox
+docker-compose stop netbox netbox-worker
-docker-compose rm -f netbox
+docker-compose rm -f netbox netbox-worker
-docker-compose up -d netbox
+docker-compose up -d netbox netbox-worker
 ```
 
+### Webhooks don't work
+
+First make sure that the webhooks feature is enabled in your Netbox configuration and that a redis host is defined.
+Check `netbox.env` if the following variables are defined:
+
+```
+WEBHOOKS_ENABLED=true
+REDIS_HOST=redis
+```
+
+Then make sure that the `redis` container and at least one `netbox-worker` are running.
+
+```
+# check the container status
+$ docker-compose ps
+
+Name                           Command               State                Ports             
+--------------------------------------------------------------------------------------------------------
+netbox-docker_netbox-worker_1   /opt/netbox/docker-entrypo ...   Up                                     
+netbox-docker_netbox_1          /opt/netbox/docker-entrypo ...   Up                                     
+netbox-docker_nginx_1           nginx -c /etc/netbox-nginx ...   Up      80/tcp, 0.0.0.0:32776->8080/tcp
+netbox-docker_postgres_1        docker-entrypoint.sh postgres    Up      5432/tcp
+netbox-docker_redis_1           docker-entrypoint.sh redis ...   Up      6379/tcp
+
+# connect to redis and send PING command:
+$ docker-compose run --rm -T redis sh -c 'redis-cli -h redis -a $REDIS_PASSWORD ping'
+Warning: Using a password with '-a' option on the command line interface may not be safe.
+PONG
+```
+
+If `redis` and the `netbox-worker` are not available, make sure you have updated your `docker-compose.yml` file!
+
+Everything's up and running? Then check the log of `netbox-worker` and/or `redis`:
+
+```bash
+docker-compose logs -f netbox-worker
+docker-compose logs -f redis
+```
+
+Still no clue? You can connect to the `redis` container and have it report any command that is currently executed on the server:
+
+```bash
+docker-compose run --rm -T redis sh -c 'redis-cli -h redis -a $REDIS_PASSWORD monitor'
+
+# Hit CTRL-C a few times to leave
+```
+
+If you don't see anything happening after you triggered a webhook, double-check the configuration of the `netbox` and the `netbox-worker` containers and also check the configuration of your webhook in the admin interface of Netbox.
+
 ### Breaking Changes
 
-From time to time it might become necessary to re-order the structure of the container.
+From time to time it might become necessary to re-engineer the structure of this setup.
-Things like the `docker-compose.yml` file or your Kubernets or OpenShift configurations have to be adjusted as a consequence.
+Things like the `docker-compose.yml` file or your Kubernetes or OpenShift configurations have to be adjusted as a consequence.
 Since April 2018 each image built from this repo contains a `NETBOX_DOCKER_PROJECT_VERSION` label.
 You can check the label of your local image by running `docker inspect ninech/netbox:v2.3.1 --format "{{json .ContainerConfig.Labels}}"`.
+Compare the version with the list below to check whether a breaking change was introduced with that version.
 
-The following is a list of breaking changes:
+The following is a list of breaking changes of the `netbox-docker` project:
 
+* 0.4.0: In order to use Netbox webhooks you need to add Redis and a netbox-worker to your docker-compose.yml.
 * 0.3.0: Field `filterable: <boolean` was replaced with field `filter_logic: loose/exact/disabled`. It will default to `CF_FILTER_LOOSE=loose` when not defined.
 * 0.2.0: Re-organized paths: `/etc/netbox -> /etc/netbox/config` and `/etc/reports -> /etc/netbox/reports`. Fixes [#54](https://github.com/ninech/netbox-docker/issues/54).
 * 0.1.0: Introduction of the `NETBOX_DOCKER_PROJECT_VERSION`. (Not a breaking change per se.)
@@ -304,9 +384,15 @@ You can use the following ENV variables to customize the build:
            Default: https://github.com/${SRC_REPO}/netbox/archive/$BRANCH.tar.gz
 ```
 
+### Publishing Docker Images
+
+New Docker Images are built and published every 24h by using travis:
+
+[![Build Status](https://travis-ci.org/ninech/netbox-docker.svg?branch=master)][travis]
+
 ## Tests
 
-To run the test coming with NetBox, use the `docker-compose.yml` file as such:
+To run the tests coming with NetBox, use the `docker-compose.yml` file as such:
 
 ```
 $ docker-compose run netbox ./manage.py test

configuration/configuration.py

@@ -139,6 +139,19 @@ PAGINATE_COUNT = int(os.environ.get('PAGINATE_COUNT', 50))
 # prefer IPv4 instead.
 PREFER_IPV4 = os.environ.get('PREFER_IPV4', 'False').lower() == 'true'
 
+# The Webhook event backend is disabled by default. Set this to True to enable it. Note that this requires a Redis
+# database be configured and accessible by NetBox (see `REDIS` below).
+WEBHOOKS_ENABLED = os.environ.get('WEBHOOKS_ENABLED', 'False').lower() == 'true'
+
+# Redis database settings (optional). A Redis database is required only if the webhooks backend is enabled.
+REDIS = {
+    'HOST': os.environ.get('REDIS_HOST', 'localhost'),
+    'PORT': os.environ.get('REDIS_PORT', '6379'),
+    'PASSWORD': os.environ.get('REDIS_PASSWORD', read_secret('redis_password')),
+    'DATABASE': os.environ.get('REDIS_DATABASE', '0'),
+    'DEFAULT_TIMEOUT': os.environ.get('REDIS_TIMEOUT', '300'),
+}
+
 # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of
 # this setting is derived from the installed location.
 REPORTS_ROOT = os.environ.get('REPORTS_ROOT', '/etc/netbox/reports')

docker-compose.yml

@@ -1,6 +1,6 @@
 version: '3'
 services:
-    netbox:
+  netbox: &netbox
     build:
       context: .
       args:
@@ -8,6 +8,8 @@ services:
     image: ninech/netbox:${VERSION-latest}
     depends_on:
     - postgres
+    - redis
+    - netbox-worker
     env_file: netbox.env
     volumes:
     - ./startup_scripts:/opt/netbox/startup_scripts:ro
@@ -17,9 +19,18 @@ services:
     - netbox-static-files:/opt/netbox/netbox/static
     - netbox-media-files:/opt/netbox/netbox/media
     - netbox-report-files:/etc/netbox/reports:ro
+  netbox-worker:
+    <<: *netbox
+    depends_on:
+    - redis
+    entrypoint:
+    - python3
+    - /opt/netbox/netbox/manage.py
+    command:
+    - rqworker
   nginx:
-        image: nginx:1.13-alpine
+    command: nginx -c /etc/netbox-nginx/nginx.conf
-        command: nginx -g 'daemon off;' -c /etc/netbox-nginx/nginx.conf
+    image: nginx:1.15-alpine
     depends_on:
     - netbox
     ports:
@@ -32,7 +43,15 @@ services:
     env_file: postgres.env
     volumes:
     - netbox-postgres-data:/var/lib/postgresql/data
-
+  redis:
+    image: redis:4-alpine
+    command:
+    - sh
+    - -c # this is to evaluate the $REDIS_PASSWORD from the env
+    - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
+    env_file: redis.env
+    volumes:
+    - netbox-redis-data:/data
 volumes:
   netbox-static-files:
     driver: local
@@ -44,3 +63,5 @@ volumes:
     driver: local
   netbox-postgres-data:
     driver: local
+  netbox-redis-data:
+    driver: local

docker/nginx.conf

@@ -1,5 +1,8 @@
+daemon off;
 worker_processes 1;
 
+error_log /dev/stderr info;
+
 events {
     worker_connections 1024;
 }
@@ -16,7 +19,6 @@ http {
 
     server {
         listen      8080;
-        server_name localhost;
         access_log  off;
 
         location /static/ {

netbox.env

@@ -13,8 +13,11 @@ NAPALM_USERNAME=
 NAPALM_PASSWORD=
 NAPALM_TIMEOUT=10
 MAX_PAGE_SIZE=0
+REDIS_HOST=redis
+REDIS_PASSWORD=H733Kdjndks81
 SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj
 SUPERUSER_NAME=admin
 SUPERUSER_EMAIL=admin@example.com
 SUPERUSER_PASSWORD=admin
 SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567
+WEBHOOKS_ENABLED=true

redis.env

@@ -0,0 +1 @@
+REDIS_PASSWORD=H733Kdjndks81