Merge branch 'main' into frontend-i18n

Dockerfile

@@ -29,8 +29,8 @@ FROM python:3.8-slim-buster as runtime-image
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV ADMIN_USER='admin'
-ENV ADMIN_PASSWORD='mediacms'
 ENV ADMIN_EMAIL='admin@localhost'
+#ENV ADMIN_PASSWORD='uncomment_and_set_password_here'
 
 # See: https://github.com/celery/celery/issues/6285#issuecomment-715316219
 ENV CELERY_APP='cms'
@@ -50,11 +50,17 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 COPY --chown=www-data:www-data --from=compile-image /home/mediacms.io /home/mediacms.io
 
 RUN apt-get update -y && apt-get -y upgrade && apt-get install --no-install-recommends \
-    supervisor nginx ffmpeg imagemagick procps -y && \
+    supervisor nginx imagemagick procps wget xz-utils -y && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get purge --auto-remove && \
     apt-get clean
 
+RUN wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \
+    mkdir -p tmp && \
+    tar -xf ffmpeg-release-amd64-static.tar.xz --strip-components 1 -C tmp && \
+    cp -v tmp/ffmpeg tmp/ffprobe tmp/qt-faststart /usr/local/bin && \
+    rm -rf tmp ffmpeg-release-amd64-static.tar.xz 
+
 WORKDIR /home/mediacms.io/mediacms
 
 EXPOSE 9000 80

README.md

@@ -97,7 +97,13 @@ There are two ways to run MediaCMS, through Docker Compose and through installin
 
 ## Configuration
 
-Several options are available on cms/settings.py, most of the things that are allowed or should be disallowed are described there. It is advisable to override any of them by adding it to cms/local_settings.py. All configuration options will be documented gradually on the [Configuration](docs/Configuration.md) page.
+Visit [Configuration](docs/Configuration.md) page.
+
+
+## Documentation
+* [Users documentation](docs/user_docs.md) page
+* [Administrators documentation](docs/admins_docs.md) page
+* [Developers documentation](docs/developers_docs.md) page
 
 
 ## Technology
@@ -122,12 +128,6 @@ If you like the project, here's a few things you can do
 - Star the project
 - Add functionality, work on a PR, fix an issue! 
 
-## Developers info
-
-- API documentation through Swagger is available under /swagger URL of your installation -example https://demo.mediacms.io/swagger/
-- We're working on proper documentation for users, managers and developers, until then checkout what's available on the docs/ folder of this repository
-- Before you send a PR, make sure your code is properly formatted. For that, use `pre-commit install` to install a pre-commit hook and run `pre-commit run --all` and fix everything before you commit. This pre-commit will check for your code lint everytime you commit a code.
-- Checkout the [Code of conduct page](CODE_OF_CONDUCT.md) if you want to contribute to this repository
 
 ## Contact
 info@mediacms.io

deploy/docker/prestart.sh

@@ -4,20 +4,29 @@ RANDOM_ADMIN_PASS=`python -c "import secrets;chars = 'abcdefghijklmnopqrstuvwxyz
 ADMIN_PASSWORD=${ADMIN_PASSWORD:-$RANDOM_ADMIN_PASS}
 
 if [ X"$ENABLE_MIGRATIONS" = X"yes" ]; then
+    echo "Running migrations service"
     python manage.py migrate
-    python manage.py loaddata fixtures/encoding_profiles.json
+    EXISTING_INSTALLATION=`echo "from users.models import User; print(User.objects.exists())" |python manage.py shell`
-    python manage.py loaddata fixtures/categories.json
+    if [ "$EXISTING_INSTALLATION" = "True" ]; then 
+        echo "Loaddata has already run"
+    else
+        echo "Running loaddata and creating admin user"
+        python manage.py loaddata fixtures/encoding_profiles.json
+        python manage.py loaddata fixtures/categories.json
+
+    	# post_save, needs redis to succeed (ie. migrate depends on redis)
+        DJANGO_SUPERUSER_PASSWORD=$ADMIN_PASSWORD python manage.py createsuperuser \
+            --no-input \
+            --username=admin \
+            --email=$ADMIN_EMAIL \
+            --database=default || true
+        echo "Created admin user with password: $ADMIN_PASSWORD"
+
+    fi
+    echo "RUNNING COLLECTSTATIC"
+
     python manage.py collectstatic --noinput
 
-    echo "Admin Password: $ADMIN_PASSWORD"
-
-    # post_save, needs redis to succeed (ie. migrate depends on redis)
-    DJANGO_SUPERUSER_PASSWORD=$ADMIN_PASSWORD python manage.py createsuperuser \
-        --no-input \
-        --username=$ADMIN_USER \
-        --email=$ADMIN_EMAIL \
-        --database=default || true
-
     # echo "Updating hostname ..."
     # TODO: Get the FRONTEND_HOST from cms/local_settings.py
     # echo "from django.contrib.sites.models import Site; Site.objects.update(name='$FRONTEND_HOST', domain='$FRONTEND_HOST')" | python manage.py shell

docker-compose-http-proxy.yaml

@@ -18,6 +18,8 @@ services:
       ENABLE_CELERY_SHORT: 'no'
       ENABLE_CELERY_LONG: 'no'
       ENABLE_CELERY_BEAT: 'no'
+    command: "./deploy/docker/prestart.sh"
+    restart: on-failure
     depends_on:
       redis:
         condition: service_healthy

docker-compose-https-proxy.yaml

@@ -20,6 +20,8 @@ services:
       ENABLE_CELERY_SHORT: 'no'
       ENABLE_CELERY_LONG: 'no'
       ENABLE_CELERY_BEAT: 'no'
+    command: "./deploy/docker/prestart.sh"
+    restart: on-failure
     depends_on:
       redis:
         condition: service_healthy

docker-compose-letsencrypt.yaml

@@ -38,6 +38,8 @@ services:
       ENABLE_CELERY_SHORT: 'no'
       ENABLE_CELERY_LONG: 'no'
       ENABLE_CELERY_BEAT: 'no'
+    command: "./deploy/docker/prestart.sh"
+    restart: on-failure
     depends_on:
       redis:
         condition: service_healthy

docker-compose-named-volumes.yaml

@@ -11,6 +11,8 @@ services:
       ENABLE_CELERY_SHORT: 'no'
       ENABLE_CELERY_LONG: 'no'
       ENABLE_CELERY_BEAT: 'no'
+    command: "./deploy/docker/prestart.sh"
+    restart: on-failure
     depends_on:
       redis:
         condition: service_healthy

docker-compose.yaml

@@ -11,6 +11,8 @@ services:
       ENABLE_CELERY_SHORT: 'no'
       ENABLE_CELERY_LONG: 'no'
       ENABLE_CELERY_BEAT: 'no'
+    command: "./deploy/docker/prestart.sh"
+    restart: on-failure
     depends_on:
       redis:
         condition: service_healthy

docs/Configuration.md

@@ -1,6 +1,6 @@
 ## Configuration
 
-A number of options are available on `cms/settings.py`. 
+Several options are available on `cms/settings.py`, most of the things that are allowed or should be disallowed are described there.
 
 It is advisable to override any of them by adding it to `local_settings.py` . 
 

docs/Docker_Compose.md

@@ -30,6 +30,13 @@ docker-compose up
 
 This will download all MediaCMS related Docker images and start all containers. Once it finishes, MediaCMS will be installed and available on http://localhost or http://ip
 
+A user admin has been created with random password, you should be able to see it at the end of migrations container, eg
+
+```
+migrations_1     | Created admin user with password: gwg1clfkwf
+```
+
+or if you have set the ADMIN_PASSWORD variable on Dockerfile, that variable will be set as the admin user's password
 
 ## Update
 

docs/admins_docs.md

@@ -0,0 +1,29 @@
+# Administrators documentation
+
+## Table of contents
+- [Manage pages](#manage-pages)
+- [Django admin dashboard](#django-admin-dashboard)
+- [On portal workflow](#on-portal-workflow)
+- [On user roles](#on-user-roles)
+- [Adding languages for Captions and subtitles](#adding-languages-for-captions-and-subtitles)
+- [Add/delete categories and tags](#adddelete-categories-and-tags)
+- [Video transcoding](#video-transcoding)
+
+
+## Manage pages
+
+## Django admin dashboard
+
+## On portal workflow
+Who can publish content, how content appears on public listings.Difference between statuses (private, unlisted, public)
+
+## On user roles
+Differences over MediaCMS manager, MediaCMS editor, logged in user
+
+## Adding languages for Captions and subtitles
+
+## Add/delete categories and tags
+Through the admin section - http://your_installation/admin/
+
+## Video transcoding
+Add / remove resolutions and profiles through http://your_installation/admin/encodeprofile

docs/developers_docs.md

@@ -0,0 +1,83 @@
+# Developers documentation
+
+## Table of contents
+- [System architecture](#system-architecture)
+- [API documentation](#api-documentation)
+- [How to contribute](#how-to-contribute)
+- [Working with Docker tips](#working-with-docker-tips)
+- [How video is transcoded](#how-video-is-transcoded)
+
+## How to contribute
+Before you send a PR, make sure your code is properly formatted. For that, use `pre-commit install` to install a pre-commit hook and run `pre-commit run --all` and fix everything before you commit. This pre-commit will check for your code lint everytime you commit a code.
+
+Checkout the [Code of conduct page](../CODE_OF_CONDUCT.md) if you want to contribute to this repository
+
+
+## System architecture
+
+## API documentation
+API is documented using Swagger - checkout ot http://your_installation/swagger - example https://demo.mediacms.io/swagger/
+This page allows you to login to perform authenticated actions - it will also use your session if logged in. 
+
+
+## Working with Docker tips
+
+To perform the Docker installation, follow instructions to install Docker + Docker compose (docs/Docker_Compose.md) and then build/start docker-compose-dev.yaml . This will run the frontend application on port 8088 on top of all other containers (including the Django web application on port 80)
+
+```
+docker-compose -f docker-compose-dev.yaml build
+docker-compose -f docker-compose-dev.yaml up
+```
+
+### Frontend application changes
+Eg change `frontend/src/static/js/pages/HomePage.tsx` , dev application refreshes in a number of seconds (hot reloading) and I see the changes, once I'm happy I can run
+
+```
+docker-compose -f docker-compose-dev.yaml -T frontend npm run dist
+```
+
+And then in order for the changes to be visible on the application while served through nginx, 
+
+```
+cp -r frontend/dist/static/* static/
+```
+
+POST calls: cannot be performed through the dev server, you have to make through the normal application (port 80) and then see changes on the dev application on port 8088. 
+Make sure the urls are set on `frontend/.env` if different than localhost
+
+
+Media page: need to upload content through the main application (nginx/port 80), and then use an id for page media.html, for example `http://localhost:8088/media.html?m=nc9rotyWP`
+
+There are some issues with CORS too to resolve, in order for some pages to function, eg the manage comments page
+
+```http://localhost:8088/manage-media.html px manage_media
+```
+
+### Backend application changes
+After I make changes to the django application (eg make a change on `files/forms.py`) in order to see the changes I have to restart the web container
+
+```
+docker-compose -f docker-compose-dev.yaml restart web
+```
+
+## How video is transcoded
+
+Original files get uploaded to the application server, and they get stored there as FileFields.
+
+If files are videos and the duration is greater than a number (defined on settings, I think 4minutes), they are also broken in chunks, so one Encode object per chunk, for all enabled EncodeProfiles.
+
+Then the workers start picking Encode objects and they transcode the chunks, so if a chunk gets transcoded correctly, the original file (the small chunk) gets replaced by the transcoded file, and the Encode object status is marked as 'success'.
+
+
+original.mp4 (1G, 720px)--> Encode1 (100MB, 240px, chunk=True), Encode2 (100MB, 240px, chunk=True)...EncodeXX (100MB, 720px, chunk=True) ---> when all Encode objects are success, for a resolution, they get concatenated to the original_resolution.mp4 file and this gets stored as Encode object (chunk=False). This is what is available for download.
+
+Apparently the Encode object is used to store Encoded files that are served eventually (chunk=False, status='success'), but also files while they are on their way to get transcoded (chunk=True, status='pending/etc')
+
+(Parenthesis opening)
+there is also an experimental small service (not commited to the repo currently) that speaks only through API and a) gets tasks to run, b) returns results. So it makes a request and receives an ffmpeg command, plus a file, it runs the ffmpeg command, and returns the result.I've used this mechanism on a number of installations to migrate existing videos through more servers/cpu and has worked with only one problem, some temporary files needed to be removed from the servers (through a periodic task, not so big problem)
+(Parenthesis closing)
+
+
+When the Encode object is marked as success and chunk=False, and thus is available for download/stream, there is a task that gets started and saves an HLS version of the file (1 mp4-->x number of small .ts chunks). This would be FILES_C
+
+This mechanism allows for workers that have access on the same filesystem (either localhost, or through a shared network filesystem, eg NFS/EFS) to work on the same time and produce results. 

docs/user_docs.md

@@ -0,0 +1,208 @@
+# Users documentation
+
+## Table of contents
+- [Uploading media](#uploading-media)
+- [Downloading media](#downloading-media)
+- [Adding captions/subtitles](#adding-captionssubtitles)
+- [Search media](#search-media)
+- [Share media](#share-media)
+- [Embed media](#embed-media)
+- [Customize my profile options](#customize-my-profile-options)
+
+## Uploading media
+
+### How to Upload Media
+
+Uploading media is as simple as clicking the _Upload Media_ button, waiting for media to upload, and then clicking the media to add metadata (title, description etc.) by filling out a form.
+
+#### Click Upload Button
+
+Click the _Upload Media_ button from the right-side of the screen at the top:
+
+<p align="left">
+    <img src="./images/Click-Upload-Media-button.png"/>
+</p>
+
+#### Upload Page
+
+Clicking the _Upload Media_ button takes you to the upload page at a URL like this:
+
+https://demo.mediacms.io/upload
+
+#### Click Browse Button
+
+Here you should click the _"Browse your files"_ button (or drag and drop a file from your desktop):
+
+<p align="left">
+    <img src="./images/Click-Browse-button.png"/>
+</p>
+
+#### Select media file and click Open button
+
+Select the media file that you want to upload and click the _"Open"_ button:
+
+<p align="left">
+    <img src="./images/Select-Media-File-Click-Open.png"/>
+</p>
+
+#### Wait for file to upload
+
+Wait for the file to finish uploading:
+
+<p align="left">
+    <img src="./images/Uploading.png"/>
+</p>
+
+#### Pause uploading
+
+If you want you can pause upload by clicking _Pause button_:
+
+<p align="left">
+    <img src="./images/Pause-button.png"/>
+</p>
+
+#### Continue uploading
+
+Continue upload by clicking _Continue button_:
+
+<p align="left">
+    <img src="./images/Continue-button.png"/>
+</p>
+
+#### Wait for media to finish Processing
+
+Wait for the media file to finish Processing:
+
+<p align="left">
+    <img src="./images/Processing.png"/>
+</p>
+
+#### Click View media button
+
+Click the View media button to open the media page:
+
+<p align="left">
+    <img src="./images/Click-View-media-button.png"/>
+</p>
+
+#### Media will be in the encoding queue
+
+The media will take some time to finish encoding (MediaCMS will transcode the file to several formats and resolutions). Meanwhile you can edit the media file to add metadata.
+
+#### Click Edit Media button
+
+Click the EDIT MEDIA button to add metadata and configure the poster image:
+
+<p align="left">
+    <img src="./images/Click-Edit-Media-button.png"/>
+</p>
+
+#### Add Metadata (Summary, Description etc.)
+
+Make sure you fill in all the required fields, and try to complete as many of the non-required fields as possible. This ensures the database is populated with useful meta-data to help others access useful information about you and your video.
+
+<p align="left">
+    <img src="./images/Edit-Media-Metadata-1.png"/>
+</p>
+
+<p align="left">
+    <img src="./images/Edit-Media-Metadata-2.png"/>
+</p>
+
+
+
+## Downloading media
+
+MediaCMS offers a configurable option whereby users can make their media files available for download. Downloads are available for transcoded files, and the original file.
+
+#### How To Enable Download
+
+Visit the media view page and choose the EDIT MEDIA button.
+
+Select the checkbox for "Allow Downloads"
+
+#### How To Download Media
+
+Visit the media view page and click the DOWNLOAD button below the video player.
+
+<p align="left">
+    <img src="./images/Click-Download-Button.png">
+</p>
+
+Choose the version you wish to download - a transcoded file or the original file:
+
+<p align="left">
+    <img src="./images/Click-version-download.png">
+</p>
+
+Choose Save File and click the OK button.
+
+<p align="left">
+    <img src="./images/Save-File.png">
+</p>
+
+
+
+## Adding captions/subtitles
+
+With MediaCMS you can add subtitles/captions to your video by uploading a subtitles file in the popular .vtt format.
+
+### Visit Media Page & Click EDIT SUBTITLE Button
+
+Visit the "single media page" for the media you wish to add subtitles/captions to and click the EDIT SUBTITLES button:
+
+<p align="left">
+    <img src="./images/Click-EDIT-SUBTITLE.png"/>
+</p>
+
+### Upload Subtitles in .vtt Format
+
+Click the Language menu to select the correct language for your Subtitles/Captions file:
+
+<p align="left">
+    <img src="./images/Click-Subtitle-Language-Menu.png"/>
+</p>
+
+Choose the correct Language for your file:
+
+<p align="left">
+    <img src="./images/Subtitles-captions1.png"/>
+</p>
+
+Click Browse to find a subtitles/captions file on your computer (if your file is not in the .vtt format, you may find a conversion service on the Internet):
+
+<p align="left">
+    <img src="./images/Subtitles-captions2.png"/>
+</p>
+
+Choose a .vtt subtitles/captions file from your computer:
+
+<p align="left">
+    <img src="./images/Subtitles-captions3.png"/>
+</p>
+
+Click the Add button to add the file to your media:
+
+<p align="left">
+    <img src="./images/Click-ADD-button.png"/>
+</p>
+
+### View Subtitles/Captions in Video Player
+
+You can now watch the captions/subtitles play back in the video player - and toggle display on/off by clicking the CC button:
+
+<p align="left">
+    <img src="./images/CC-display.png"/>
+</p>
+
+## Search media
+How search can be used
+
+## Share media
+How to share media
+
+## Embed media
+How to use the embed media option
+
+## Customize my profile options
+Customize profile and channel

install.sh

@@ -21,21 +21,24 @@ It is expected to run on a new system **with no running instances of any these s
 done
 
 
-if [[ `lsb_release -d` == *"Ubuntu 20"* ]]; then
+osVersion=$(lsb_release -d)
+if [[ $osVersion == *"Ubuntu 20"* ]] || [[ $osVersion == *"Ubuntu 18"* ]] || [[ $osVersion == *"buster"* ]]; then
     echo 'Performing system update and dependency installation, this will take a few minutes'
-    apt-get update && apt-get -y upgrade && apt install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip ffmpeg imagemagick python3-certbot-nginx certbot wget -y
+    apt-get update && apt-get -y upgrade && apt-get install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip imagemagick python3-certbot-nginx certbot wget xz-utils -y
-elif [[ `lsb_release -d`  = *"Ubuntu 18"* ]]; then
-    echo 'Performing system update and dependency installation, this will take a few minutes'
-    apt-get update && apt-get -y upgrade && apt install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip ffmpeg imagemagick python3-certbot-nginx certbot wget -y
-# added check for Debian 10 (buster)
-elif [[ `lsb_release -d` == *"buster"* ]]; then
-    echo 'Performing system update and dependency installation, this will take a few minutes'
-    apt-get update && apt-get -y upgrade && apt install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip ffmpeg imagemagick python3-certbot-nginx certbot wget -y
 else
     echo "This script is tested for Ubuntu 18 and 20 versions only, if you want to try MediaCMS on another system you have to perform the manual installation"
-exit
+    exit
 fi
 
+# install ffmpeg
+echo "Downloading and installing ffmpeg"
+wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
+mkdir -p tmp
+tar -xf ffmpeg-release-amd64-static.tar.xz --strip-components 1 -C tmp
+cp -v tmp/{ffmpeg,ffprobe,qt-faststart} /usr/local/bin
+rm -rf tmp ffmpeg-release-amd64-static.tar.xz
+echo "ffmpeg installed to /usr/local/bin"
+
 read -p "Enter portal URL, or press enter for localhost : " FRONTEND_HOST
 read -p "Enter portal name, or press enter for 'MediaCMS : " PORTAL_NAME