new https proxy based on haproxy, also proxying TURN

This commit is contained in:
chandi 2024-11-25 00:35:18 +01:00
parent 325690e0c1
commit ca0a159cc0
15 changed files with 181 additions and 254 deletions

View File

@ -31,8 +31,10 @@ services:
WELCOME_MESSAGE: ${WELCOME_MESSAGE:-}
WELCOME_FOOTER: ${WELCOME_FOOTER}
STUN_SERVER: stun:${STUN_IP}:${STUN_PORT}
TURN_SERVER: ${TURN_SERVER:-}
ENABLE_HTTPS_PROXY: ${ENABLE_HTTPS_PROXY:-false}
TURN_SECRET: ${TURN_SECRET:-}
TURN_EXT_SERVER: ${TURN_EXT_SERVER:-}
TURN_EXT_SECRET: ${TURN_EXT_SECRET:-}
ENABLE_LEARNING_DASHBOARD: ${ENABLE_LEARNING_DASHBOARD:-true}
NUMBER_OF_BACKEND_NODEJS_PROCESSES: {{ .Env.NUMBER_OF_BACKEND_NODEJS_PROCESSES }}
volumes:
@ -406,32 +408,20 @@ services:
{{end}}
{{ if isTrue .Env.ENABLE_HTTPS_PROXY }}
# https
https_proxy:
image: valian/docker-nginx-auto-ssl
restart: unless-stopped
haproxy:
build: mod/haproxy
image: alangecker/bbb-haproxy:2.8.10
volumes:
- ssl_data:/etc/resty-auto-ssl
{{ if .Env.EXTERNAL_IPv6 }}
- ./mod/https/site.conf:/etc/nginx/conf.d/bbb-docker.conf
{{else}}
- ./mod/https/site-ipv4only.conf:/etc/nginx/conf.d/bbb-docker.conf
{{end}}
{{ if isTrue .Env.DEV_MODE }}
# allow bbb api access without https
- ./mod/https/force-https.conf:/usr/local/openresty/nginx/conf/force-https.conf
{{end}}
- ./data/haproxy/letsencrypt:/etc/letsencrypt
- ./mod/haproxy/haproxy.cfg:/etc/haproxy/haproxy.cfg
environment:
{{ if isTrue .Env.DEV_MODE }}
ALLOWED_DOMAINS: ""
{{else}}
ALLOWED_DOMAINS: ${DOMAIN}
{{end}}
RESOLVER_ADDRESS: ${RESOLVER_ADDRESS:-9.9.9.9}
- CERT1=${DOMAIN}
- STAGING=true
- EMAIL=test@chandi.it
network_mode: host
{{end}}
{{ if isTrue .Env.ENABLE_COTURN }}
# coturn
coturn:
image: coturn/coturn:4.6-alpine
@ -440,20 +430,10 @@ services:
- "--external-ip=${EXTERNAL_IPv4}/${EXTERNAL_IPv4}"
- "--external-ip=${EXTERNAL_IPv6:-::1}/${EXTERNAL_IPv6:-::1}"
- "--static-auth-secret=${TURN_SECRET}"
- "--allowed-peer-ip=${EXTERNAL_IPv4}"
volumes:
{{ if isTrue .Env.ENABLE_HTTPS_PROXY }}
- ssl_data:/etc/resty-auto-ssl
{{else}}
- ${COTURN_TLS_CERT_PATH}:/tmp/cert.pem
- ${COTURN_TLS_KEY_PATH}:/tmp/key.pem
{{end}}
- ./mod/coturn/entrypoint.sh:/usr/local/bin/docker-entrypoint.sh
- ./mod/coturn/turnserver.conf:/etc/coturn/turnserver.conf
environment:
ENABLE_HTTPS_PROXY:
user: root
network_mode: host
{{end}}
{{ if isTrue .Env.ENABLE_GREENLIGHT }}
@ -526,12 +506,6 @@ services:
entrypoint: sh -c 'echo "BIGBLUEBUTTON_RELEASE=2.7.3" > /etc/bigbluebutton/bigbluebutton-release && python server.py'
{{end}}
volumes:
{{ if isTrue .Env.ENABLE_HTTPS_PROXY }}
ssl_data:
{{end}}
networks:
bbb-net:
ipam:

View File

@ -16,7 +16,7 @@ RUN groupadd -g 998 bigbluebutton \
&& chown bigbluebutton:bigbluebutton /etc/bigbluebutton
# add dockerize
ENV DOCKERIZE_VERSION v0.6.1
ENV DOCKERIZE_VERSION v0.7.0
RUN wget -q https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
&& tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
&& rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz

View File

@ -8,10 +8,26 @@
<constructor-arg index="0" value="{{ .Env.STUN_SERVER }}"/>
</bean>
{{if .Env.TURN_SERVER }}
<bean id="turn0" class="org.bigbluebutton.web.services.turn.TurnServer">
<constructor-arg index="0" value="{{ .Env.TURN_SECRET }}"/>
<constructor-arg index="1" value="{{ .Env.TURN_SERVER }}"/>
<bean id="turn0" class="org.bigbluebutton.web.services.turn.TurnServer">
<constructor-arg index="0" value="{{ .Env.TURN_SECRET }}"/>
<constructor-arg index="1" value="turn:{{ .Env.DOMAIN }}:3478"/>
<constructor-arg index="2" value="86400"/>
</bean>
{{if and (isTrue .Env.ENABLE_HTTPS_PROXY) (ne .Env.DOMAIN "10.7.7.1") }}
{{/* ignore when using a self signed certificate in dev mode */}}
<bean id="turn1" class="org.bigbluebutton.web.services.turn.TurnServer">
<constructor-arg index="0" value="{{ .Env.TURN_SECRET }}"/>
<constructor-arg index="1" value="turns:{{ .Env.DOMAIN }}:443?transport=tcp"/>
<constructor-arg index="2" value="86400"/>
</bean>
{{end}}
{{if .Env.TURN_EXT_SERVER }}
<bean id="turn2" class="org.bigbluebutton.web.services.turn.TurnServer">
<constructor-arg index="0" value="{{ .Env.TURN_EXT_SECRET }}"/>
<constructor-arg index="1" value="{{ .Env.TURN_EXT_SERVER }}"/>
<constructor-arg index="2" value="86400"/>
</bean>
{{end}}
@ -24,8 +40,14 @@
</property>
<property name="turnServers">
<set>
{{if .Env.TURN_SERVER }}
<ref bean="turn0" />
{{if and (isTrue .Env.ENABLE_HTTPS_PROXY) (ne .Env.DOMAIN "10.7.7.1") }}
<ref bean="turn1" />
{{end}}
{{if .Env.TURN_EXT_SERVER }}
<ref bean="turn2" />
{{end}}
</set>
</property>

View File

@ -1,31 +0,0 @@
#!/bin/sh
set -e
apk add jq su-exec
if [ "$ENABLE_HTTPS_PROXY" == true ]; then
while [ ! -f /etc/resty-auto-ssl/storage/file/*latest ]
do
echo "ERROR: certificate doesn't exist yet."
echo "Certificate gets create on the first request to the HTTPS proxy."
echo "We will try again..."
sleep 10
done
# extract cert
cat /etc/resty-auto-ssl/storage/file/*%3Alatest | jq -r '.fullchain_pem' > /tmp/cert.pem
cat /etc/resty-auto-ssl/storage/file/*%3Alatest | jq -r '.privkey_pem' > /tmp/key.pem
fi
if [ ! -f /tmp/cert.pem ] || [ ! -f /tmp/key.pem ]; then
echo "ERROR: certificate not found, but coturn relies on it."
echo "Use either auto HTTPS proxy or"
echo "provide path to certificates in .env file"
exit 1
fi
# If command starts with an option, prepend with turnserver binary.
if [ "${1:0:1}" == '-' ]; then
set -- turnserver "$@"
fi
su-exec nobody $(eval "echo $@")

View File

@ -1,73 +1,29 @@
# Example coturn configuration for BigBlueButton
# These are the two network ports used by the TURN server which the client
# may connect to. We enable the standard unencrypted port 3478 for STUN,
listening-port=3478
# and since TLS over SMTP port (465) is now blocked by major browser vendors,
# we reverted to the most common coturn TLS port 5349, which has limitations
# in restrictive firewall environments. For maximum client support run
# coturn on a dedicated host on port 443.
tls-listening-port=5349
# listening-ip=${INTERNAL_IP:-$IP}
# relay-ip=${INTERNAL_IP:-$IP}
# If the server has multiple IP addresses, you may wish to limit which
# addresses coturn is using. Do that by setting this option (it can be
# specified multiple times). The default is to listen on all addresses.
# You do not normally need to set this option.
#listening-ip=172.17.19.101
min-port=32769
max-port=65535
# verbose
# If the server is behind NAT, you need to specify the external IP address.
# If there is only one external address, specify it like this:
#external-ip=172.17.19.120
# If you have multiple external addresses, you have to specify which
# internal address each corresponds to, like this. The first address is the
# external ip, and the second address is the corresponding internal IP.
#external-ip=172.17.19.131/10.0.0.11
#external-ip=172.17.18.132/10.0.0.12
# Fingerprints in TURN messages are required for WebRTC
fingerprint
# The long-term credential mechanism is required for WebRTC
lt-cred-mech
# Configure coturn to use the "TURN REST API" method for validating time-
# limited credentials. BigBlueButton will generate credentials in this
# format. Note that the static-auth-secret value specified here must match
# the configuration in BigBlueButton's turn-stun-servers.xml
# You can generate a new random value by running the command:
# openssl rand -hex 16
use-auth-secret
# static-auth-secret=<random value>
realm=bbb-docker
# If the realm value is unspecified, it defaults to the TURN server hostname.
# You probably want to configure it to a domain name that you control to
# improve log output. There is no functional impact.
realm=example.com
keep-address-family
# Configure TLS support.
# Adjust these paths to match the locations of your certificate files
cert=/tmp/cert.pem
pkey=/tmp/key.pem
# Limit the allowed ciphers to improve security
# Based on https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
cipher-list="ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS"
# Enable longer DH TLS key to improve security
dh2066
# All WebRTC-compatible web browsers support TLS 1.2 or later, so disable
# older protocols
no-cli
no-tlsv1
no-tlsv1_1
# To enable single filename logs you need to enable the simple-log flag
syslog
#verbose
# Block connections to IP ranges which shouldn't be reachable
no-loopback-peers
no-multicast-peers
# Allocate Address Family according
# If enabled then TURN server allocates address family according the TURN
# Client <=> Server communication address family.
# (By default Coturn works according RFC 6156.)
# !!Warning: Enabling this option breaks RFC6156 section-4.2 (violates use default IPv4)!!
keep-address-family
# we only need to allow peer connections from the machine itself (from mediasoup or freeswitch).
denied-peer-ip=0.0.0.0-255.255.255.255
denied-peer-ip=::-ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff
allowed-peer-ip=10.7.7.1

4
mod/haproxy/Dockerfile Normal file
View File

@ -0,0 +1,4 @@
FROM ghcr.io/tomdess/docker-haproxy-certbot:2.8.10
# overwrite bootstrap.sh
COPY bootstrap.sh /bootstrap.sh

26
mod/haproxy/bootstrap.sh Executable file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -e
# save container environment variables to use it
# in cron scripts
declare -p | grep -Ev '^declare -[[:alpha:]]*r' > /container.env
if [ "x$CERT1" = "x10.7.7.1" ]; then
# use self signed certificate
if [ ! -f /etc/haproxy/certs/haproxy-10.7.7.1.pem ]; then
mkdir -p /etc/haproxy/certs
# generate self signed certificate
openssl req -x509 -nodes -days 700 -newkey rsa:2048 \
-keyout /tmp/domain.key -out /tmp/domain.crt \
-subj "/C=CA/ST=Quebec/L=Montreal/O=BigBlueButton Development/OU=bbb-docker/CN=10.7.7.1"
cat /tmp/domain.key /tmp/domain.crt | tee /etc/haproxy/certs/haproxy-10.7.7.1.pem >/dev/null
fi
else
# obtain certificates from lets encrypt
/certs.sh
fi
supervisord -c /etc/supervisord.conf -n

80
mod/haproxy/haproxy.cfg Normal file
View File

@ -0,0 +1,80 @@
global
log stdout format raw local0 debug
maxconn 20480
############# IMPORTANT #################################
## DO NOT SET CHROOT OTHERWISE YOU HAVE TO CHANGE THE ##
## acme-http01-webroot.lua file ##
# chroot /jail ##
#########################################################
lua-load /etc/haproxy/acme-http01-webroot.lua
#
# SSL options
ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
ssl-default-bind-options ssl-min-ver TLSv1.2
tune.ssl.default-dh-param 4096
# workaround for bug #14 (Cert renewal blocks HAProxy indefinitely with Websocket connections)
hard-stop-after 3s
# DNS runt-time resolution on backend hosts
resolvers docker
nameserver dns "127.0.0.11:53"
defaults
log global
mode http
timeout connect 5000ms
timeout client 50000ms
timeout server 50000ms
# option forwardfor
option httplog
option dontlognull
timeout connect 5000
timeout client 50000
timeout server 50000
# never fail on address resolution
default-server init-addr last,libc,none
frontend http
bind *:80,[::]:80
mode http
acl url_acme_http01 path_beg /.well-known/acme-challenge/
http-request use-service lua.acme-http01 if METH_GET url_acme_http01
redirect scheme https code 301 if !{ ssl_fc }
frontend nginx_or_turn
bind *:443,:::443 ssl crt /etc/haproxy/certs/ ssl-min-ver TLSv1.2 alpn h2,http/1.1,stun.turn
mode tcp
option tcplog
tcp-request content capture req.payload(0,1) len 1
log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq captured_user:%{+X}[capture.req.hdr(0)]"
tcp-request inspect-delay 30s
# We terminate SSL on haproxy. HTTP2 is a binary protocol. haproxy has to
# decide which protocol is spoken. This is negotiated by ALPN.
#
# Depending on the ALPN value traffic is redirected to either port 82 (HTTP2,
# ALPN value h2) or 81 (HTTP 1.0 or HTTP 1.1, ALPN value http/1.1 or no value)
# If no ALPN value is set, the first byte is inspected and depending on the
# value traffic is sent to either port 81 or coturn.
use_backend nginx-http2 if { ssl_fc_alpn h2 }
use_backend nginx if { ssl_fc_alpn http/1.1 }
use_backend turn if { ssl_fc_alpn stun.turn }
# use_backend %[capture.req.hdr(0),map_str(/etc/haproxy/protocolmap,turn)]
default_backend turn
backend turn
mode tcp
server localhost 10.7.7.1:3478 check
backend nginx
mode tcp
server localhost 10.7.7.1:48081 send-proxy check
backend nginx-http2
mode tcp
server localhost 10.7.7.1:48082 send-proxy check

View File

@ -1,15 +0,0 @@
# overwriting force-https.conf from valian/docker-nginx-auto-ssl
location /bigbluebutton/api/join {
return 301 https://$host$request_uri;
}
# allow /api calls without redirecting to https
location /bigbluebutton/ {
proxy_pass https://127.0.0.1:443;
proxy_ssl_verify off;
}
location / {
return 301 https://$host$request_uri;
}

View File

@ -1,33 +0,0 @@
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen 443 ssl http2 default_server;
# we at still serve https via IPv6 for the
# case that an AAAA record is set.
listen [::]:443 ssl http2 default_server;
server_name _;
include resty-server-https.conf;
location / {
proxy_http_version 1.1;
proxy_pass http://127.0.0.1:48087;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_cache_bypass $http_upgrade;
proxy_read_timeout 6h;
proxy_send_timeout 6h;
client_body_timeout 6h;
send_timeout 6h;
}
}

View File

@ -1,33 +0,0 @@
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
map $remote_addr $endpoint_addr {
"~:" [::1];
default 127.0.0.1;
}
server {
listen 443 ssl http2 default_server;
listen [::]:443 ssl http2 default_server;
server_name _;
include resty-server-https.conf;
location / {
proxy_http_version 1.1;
proxy_pass http://$endpoint_addr:48087;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_cache_bypass $http_upgrade;
proxy_read_timeout 6h;
proxy_send_timeout 6h;
client_body_timeout 6h;
send_timeout 6h;
}
}

View File

@ -1,6 +1,12 @@
server {
# proxied from HAProxy
listen 48082 http2 proxy_protocol;
listen 48081 proxy_protocol;
# optional ports for other reverse proxies
listen 48087 default_server;
listen [::]:48087 default_server;
server_name _;
access_log /dev/stdout;
absolute_redirect off;

View File

@ -9,13 +9,6 @@ ENABLE_HTTPS_PROXY=true
# If your network doesn't allow access to DNS at 8.8.8.8 specify your own resolvers
#RESOLVER_ADDRESS=x.x.x.x
# coturn (a TURN Server)
# requires either the abhove HTTPS Proxy to be enabled
# or TLS certificates to be mounted to container
ENABLE_COTURN=true
#COTURN_TLS_CERT_PATH=
#COTURN_TLS_KEY_PATH=
# Greenlight Frontend
# https://docs.bigbluebutton.org/greenlight/gl-overview.html
ENABLE_GREENLIGHT=true
@ -49,6 +42,7 @@ ETHERPAD_API_KEY=SuperEtherpadKey
RAILS_SECRET=SuperRailsSecret_SuperRailsSecret
POSTGRESQL_SECRET=SuperPostgresSecret
FSESL_PASSWORD=SuperFreeswitchESLPassword
TURN_SECRET=SuperTurnSecret
@ -68,8 +62,8 @@ STUN_PORT=3478
# TURN SERVER
# uncomment and adjust following two lines to add an external TURN server
#TURN_SERVER=turns:turn.example.com:443?transport=tcp
#TURN_SECRET=
#TURN_EXT_SERVER=turns:example.org:443?transport=tcp
#TURN_EXT_SECRET=
# Allowed SIP IPs
# due to high traffic caused by bots, by default the SIP port is blocked.

View File

@ -18,20 +18,6 @@ if [ -z "$EXTERNAL_IPv4" ]; then
exit 1
fi
if [ "$ENABLE_COTURN" == true ]; then
if [ -z "$ENABLE_HTTPS_PROXY" ] && [ -z "$COTURN_TLS_CERT_PATH" ]; then
echo "ERROR: coturn requires TLS certificates."
echo "Either enable the https proxy for certificate retrival"
echo "or provide a path to your certificates in .env file."
exit 1
fi
if [ -z "$ENABLE_HTTPS_PROXY" ] && [ "$DEV_MODE" == true ]; then
echo "ERROR: the https proxy can't get a certificate if ran locally and therefor coturn will never start"
echo "you should disable coturn in .env"
exit 1
fi
fi
function get_tag {
# is submodule checked out?
if [ -f "$1/.git" ]; then
@ -63,7 +49,6 @@ docker run \
-e ENABLE_RECORDING=${ENABLE_RECORDING:-false} \
-e ENABLE_HTTPS_PROXY=${ENABLE_HTTPS_PROXY:-false} \
-e ENABLE_WEBHOOKS=${ENABLE_WEBHOOKS:-false} \
-e ENABLE_COTURN=${ENABLE_COTURN:-false} \
-e ENABLE_GREENLIGHT=${ENABLE_GREENLIGHT:-false} \
-e ENABLE_PROMETHEUS_EXPORTER=${ENABLE_PROMETHEUS_EXPORTER:-false} \
-e ENABLE_PROMETHEUS_EXPORTER_OPTIMIZATION=${ENABLE_PROMETHEUS_EXPORTER_OPTIMIZATION:-false} \

View File

@ -33,10 +33,6 @@ while [[ ! $https_proxy =~ ^(y|n)$ ]]; do
read -p "Should an automatic HTTPS Proxy be included? (y/n): " https_proxy
done
coturn=""
while [[ ! $coturn =~ ^(y|n)$ ]]; do
read -p "Should a coturn be included? (y/n): " coturn
done
if [ "$coturn" == "y" ] && [ ! "$https_proxy" == "y" ]
then
echo "Coturn needs TLS to function properly."
@ -136,6 +132,7 @@ cp sample.env .env
sed -i "s/EXTERNAL_IPv4=.*/EXTERNAL_IPv4=$EXTERNAL_IPv4/" .env
sed -i "s/EXTERNAL_IPv6=.*/EXTERNAL_IPv6=$EXTERNAL_IPv6/" .env
sed -i "s/DOMAIN=.*/DOMAIN=$DOMAIN/" .env
sed -i "s/.*STUN_IP=.*/STUN_IP=$EXTERNAL_IPv4/" .env
if [ ! "$greenlight" == "y" ]
then
@ -158,15 +155,6 @@ then
sed -i "s/#RECORDING_MAX_AGE_DAYS=.*/RECORDING_MAX_AGE_DAYS=$recording_max_age_days/" .env
fi
if [ "$coturn" == "y" ]
then
sed -i "s/.*TURN_SERVER=.*/TURN_SERVER=turns:$DOMAIN:5349?transport=tcp/" .env
TURN_SECRET=$(head /dev/urandom | tr -dc A-Za-f0-9 | head -c 32)
sed -i "s/.*TURN_SECRET=.*/TURN_SECRET=$TURN_SECRET/" .env
sed -i "s/.*STUN_IP=.*/STUN_IP=$EXTERNAL_IPv4/" .env
else
sed -i "s/ENABLE_COTURN.*/#ENABLE_COTURN=true/" .env
fi
if [ -n "$CERTPATH" ] && [ -n "$KEYPATH" ]
then
@ -190,12 +178,16 @@ RANDOM_2=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 40)
RANDOM_3=$(head /dev/urandom | tr -dc a-f0-9 | head -c 128)
RANDOM_4=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 40)
RANDOM_5=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 40)
TURN_SECRET=$(head /dev/urandom | tr -dc A-Za-f0-9 | head -c 32)
sed -i "s/SHARED_SECRET=.*/SHARED_SECRET=$RANDOM_1/" .env
sed -i "s/ETHERPAD_API_KEY=.*/ETHERPAD_API_KEY=$RANDOM_2/" .env
sed -i "s/RAILS_SECRET=.*/RAILS_SECRET=$RANDOM_3/" .env
sed -i "s/FSESL_PASSWORD=.*/FSESL_PASSWORD=$RANDOM_4/" .env
sed -i "s/POSTGRESQL_SECRET=.*/POSTGRESQL_SECRET=$RANDOM_5/" .env
sed -i "s/.*TURN_SECRET=.*/TURN_SECRET=$TURN_SECRET/" .env
./scripts/generate-compose