diff --git a/Dockerfile b/Dockerfile index 4ec6a7a..197433d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,8 +29,8 @@ FROM python:3.8-slim-buster as runtime-image ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 ENV ADMIN_USER='admin' -ENV ADMIN_PASSWORD='mediacms' ENV ADMIN_EMAIL='admin@localhost' +#ENV ADMIN_PASSWORD='uncomment_and_set_password_here' # See: https://github.com/celery/celery/issues/6285#issuecomment-715316219 ENV CELERY_APP='cms' @@ -50,11 +50,17 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY --chown=www-data:www-data --from=compile-image /home/mediacms.io /home/mediacms.io RUN apt-get update -y && apt-get -y upgrade && apt-get install --no-install-recommends \ - supervisor nginx ffmpeg imagemagick procps -y && \ + supervisor nginx imagemagick procps wget xz-utils -y && \ rm -rf /var/lib/apt/lists/* && \ apt-get purge --auto-remove && \ apt-get clean +RUN wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \ + mkdir -p tmp && \ + tar -xf ffmpeg-release-amd64-static.tar.xz --strip-components 1 -C tmp && \ + cp -v tmp/ffmpeg tmp/ffprobe tmp/qt-faststart /usr/local/bin && \ + rm -rf tmp ffmpeg-release-amd64-static.tar.xz + WORKDIR /home/mediacms.io/mediacms EXPOSE 9000 80 diff --git a/README.md b/README.md index d386e86..a39a7a2 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,13 @@ There are two ways to run MediaCMS, through Docker Compose and through installin ## Configuration -Several options are available on cms/settings.py, most of the things that are allowed or should be disallowed are described there. It is advisable to override any of them by adding it to cms/local_settings.py. All configuration options will be documented gradually on the [Configuration](docs/Configuration.md) page. +Visit [Configuration](docs/Configuration.md) page. + + +## Documentation +* [Users documentation](docs/user_docs.md) page +* [Administrators documentation](docs/admins_docs.md) page +* [Developers documentation](docs/developers_docs.md) page ## Technology @@ -122,12 +128,6 @@ If you like the project, here's a few things you can do - Star the project - Add functionality, work on a PR, fix an issue! -## Developers info - -- API documentation through Swagger is available under /swagger URL of your installation -example https://demo.mediacms.io/swagger/ -- We're working on proper documentation for users, managers and developers, until then checkout what's available on the docs/ folder of this repository -- Before you send a PR, make sure your code is properly formatted. For that, use `pre-commit install` to install a pre-commit hook and run `pre-commit run --all` and fix everything before you commit. This pre-commit will check for your code lint everytime you commit a code. -- Checkout the [Code of conduct page](CODE_OF_CONDUCT.md) if you want to contribute to this repository ## Contact info@mediacms.io \ No newline at end of file diff --git a/deploy/docker/prestart.sh b/deploy/docker/prestart.sh index f40b71a..a512792 100755 --- a/deploy/docker/prestart.sh +++ b/deploy/docker/prestart.sh @@ -4,20 +4,29 @@ RANDOM_ADMIN_PASS=`python -c "import secrets;chars = 'abcdefghijklmnopqrstuvwxyz ADMIN_PASSWORD=${ADMIN_PASSWORD:-$RANDOM_ADMIN_PASS} if [ X"$ENABLE_MIGRATIONS" = X"yes" ]; then + echo "Running migrations service" python manage.py migrate - python manage.py loaddata fixtures/encoding_profiles.json - python manage.py loaddata fixtures/categories.json + EXISTING_INSTALLATION=`echo "from users.models import User; print(User.objects.exists())" |python manage.py shell` + if [ "$EXISTING_INSTALLATION" = "True" ]; then + echo "Loaddata has already run" + else + echo "Running loaddata and creating admin user" + python manage.py loaddata fixtures/encoding_profiles.json + python manage.py loaddata fixtures/categories.json + + # post_save, needs redis to succeed (ie. migrate depends on redis) + DJANGO_SUPERUSER_PASSWORD=$ADMIN_PASSWORD python manage.py createsuperuser \ + --no-input \ + --username=admin \ + --email=$ADMIN_EMAIL \ + --database=default || true + echo "Created admin user with password: $ADMIN_PASSWORD" + + fi + echo "RUNNING COLLECTSTATIC" + python manage.py collectstatic --noinput - echo "Admin Password: $ADMIN_PASSWORD" - - # post_save, needs redis to succeed (ie. migrate depends on redis) - DJANGO_SUPERUSER_PASSWORD=$ADMIN_PASSWORD python manage.py createsuperuser \ - --no-input \ - --username=$ADMIN_USER \ - --email=$ADMIN_EMAIL \ - --database=default || true - # echo "Updating hostname ..." # TODO: Get the FRONTEND_HOST from cms/local_settings.py # echo "from django.contrib.sites.models import Site; Site.objects.update(name='$FRONTEND_HOST', domain='$FRONTEND_HOST')" | python manage.py shell diff --git a/docker-compose-http-proxy.yaml b/docker-compose-http-proxy.yaml index 11df078..8ae41b0 100644 --- a/docker-compose-http-proxy.yaml +++ b/docker-compose-http-proxy.yaml @@ -18,6 +18,8 @@ services: ENABLE_CELERY_SHORT: 'no' ENABLE_CELERY_LONG: 'no' ENABLE_CELERY_BEAT: 'no' + command: "./deploy/docker/prestart.sh" + restart: on-failure depends_on: redis: condition: service_healthy diff --git a/docker-compose-https-proxy.yaml b/docker-compose-https-proxy.yaml index b0e8793..59475e2 100644 --- a/docker-compose-https-proxy.yaml +++ b/docker-compose-https-proxy.yaml @@ -20,6 +20,8 @@ services: ENABLE_CELERY_SHORT: 'no' ENABLE_CELERY_LONG: 'no' ENABLE_CELERY_BEAT: 'no' + command: "./deploy/docker/prestart.sh" + restart: on-failure depends_on: redis: condition: service_healthy diff --git a/docker-compose-letsencrypt.yaml b/docker-compose-letsencrypt.yaml index cd03b97..3ddde6c 100644 --- a/docker-compose-letsencrypt.yaml +++ b/docker-compose-letsencrypt.yaml @@ -38,6 +38,8 @@ services: ENABLE_CELERY_SHORT: 'no' ENABLE_CELERY_LONG: 'no' ENABLE_CELERY_BEAT: 'no' + command: "./deploy/docker/prestart.sh" + restart: on-failure depends_on: redis: condition: service_healthy diff --git a/docker-compose-named-volumes.yaml b/docker-compose-named-volumes.yaml index 52f8229..63273d4 100644 --- a/docker-compose-named-volumes.yaml +++ b/docker-compose-named-volumes.yaml @@ -11,6 +11,8 @@ services: ENABLE_CELERY_SHORT: 'no' ENABLE_CELERY_LONG: 'no' ENABLE_CELERY_BEAT: 'no' + command: "./deploy/docker/prestart.sh" + restart: on-failure depends_on: redis: condition: service_healthy diff --git a/docker-compose.yaml b/docker-compose.yaml index da35826..a8c3d55 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,6 +11,8 @@ services: ENABLE_CELERY_SHORT: 'no' ENABLE_CELERY_LONG: 'no' ENABLE_CELERY_BEAT: 'no' + command: "./deploy/docker/prestart.sh" + restart: on-failure depends_on: redis: condition: service_healthy diff --git a/docs/Configuration.md b/docs/Configuration.md index 75b3a03..158b634 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -1,6 +1,6 @@ ## Configuration -A number of options are available on `cms/settings.py`. +Several options are available on `cms/settings.py`, most of the things that are allowed or should be disallowed are described there. It is advisable to override any of them by adding it to `local_settings.py` . diff --git a/docs/Docker_Compose.md b/docs/Docker_Compose.md index 68ef47c..58ee066 100644 --- a/docs/Docker_Compose.md +++ b/docs/Docker_Compose.md @@ -30,6 +30,13 @@ docker-compose up This will download all MediaCMS related Docker images and start all containers. Once it finishes, MediaCMS will be installed and available on http://localhost or http://ip +A user admin has been created with random password, you should be able to see it at the end of migrations container, eg + +``` +migrations_1 | Created admin user with password: gwg1clfkwf +``` + +or if you have set the ADMIN_PASSWORD variable on Dockerfile, that variable will be set as the admin user's password ## Update diff --git a/docs/admins_docs.md b/docs/admins_docs.md new file mode 100644 index 0000000..f14e139 --- /dev/null +++ b/docs/admins_docs.md @@ -0,0 +1,29 @@ +# Administrators documentation + +## Table of contents +- [Manage pages](#manage-pages) +- [Django admin dashboard](#django-admin-dashboard) +- [On portal workflow](#on-portal-workflow) +- [On user roles](#on-user-roles) +- [Adding languages for Captions and subtitles](#adding-languages-for-captions-and-subtitles) +- [Add/delete categories and tags](#adddelete-categories-and-tags) +- [Video transcoding](#video-transcoding) + + +## Manage pages + +## Django admin dashboard + +## On portal workflow +Who can publish content, how content appears on public listings.Difference between statuses (private, unlisted, public) + +## On user roles +Differences over MediaCMS manager, MediaCMS editor, logged in user + +## Adding languages for Captions and subtitles + +## Add/delete categories and tags +Through the admin section - http://your_installation/admin/ + +## Video transcoding +Add / remove resolutions and profiles through http://your_installation/admin/encodeprofile \ No newline at end of file diff --git a/docs/developers_docs.md b/docs/developers_docs.md new file mode 100644 index 0000000..9d619fd --- /dev/null +++ b/docs/developers_docs.md @@ -0,0 +1,83 @@ +# Developers documentation + +## Table of contents +- [System architecture](#system-architecture) +- [API documentation](#api-documentation) +- [How to contribute](#how-to-contribute) +- [Working with Docker tips](#working-with-docker-tips) +- [How video is transcoded](#how-video-is-transcoded) + +## How to contribute +Before you send a PR, make sure your code is properly formatted. For that, use `pre-commit install` to install a pre-commit hook and run `pre-commit run --all` and fix everything before you commit. This pre-commit will check for your code lint everytime you commit a code. + +Checkout the [Code of conduct page](../CODE_OF_CONDUCT.md) if you want to contribute to this repository + + +## System architecture + +## API documentation +API is documented using Swagger - checkout ot http://your_installation/swagger - example https://demo.mediacms.io/swagger/ +This page allows you to login to perform authenticated actions - it will also use your session if logged in. + + +## Working with Docker tips + +To perform the Docker installation, follow instructions to install Docker + Docker compose (docs/Docker_Compose.md) and then build/start docker-compose-dev.yaml . This will run the frontend application on port 8088 on top of all other containers (including the Django web application on port 80) + +``` +docker-compose -f docker-compose-dev.yaml build +docker-compose -f docker-compose-dev.yaml up +``` + +### Frontend application changes +Eg change `frontend/src/static/js/pages/HomePage.tsx` , dev application refreshes in a number of seconds (hot reloading) and I see the changes, once I'm happy I can run + +``` +docker-compose -f docker-compose-dev.yaml -T frontend npm run dist +``` + +And then in order for the changes to be visible on the application while served through nginx, + +``` +cp -r frontend/dist/static/* static/ +``` + +POST calls: cannot be performed through the dev server, you have to make through the normal application (port 80) and then see changes on the dev application on port 8088. +Make sure the urls are set on `frontend/.env` if different than localhost + + +Media page: need to upload content through the main application (nginx/port 80), and then use an id for page media.html, for example `http://localhost:8088/media.html?m=nc9rotyWP` + +There are some issues with CORS too to resolve, in order for some pages to function, eg the manage comments page + +```http://localhost:8088/manage-media.html px manage_media +``` + +### Backend application changes +After I make changes to the django application (eg make a change on `files/forms.py`) in order to see the changes I have to restart the web container + +``` +docker-compose -f docker-compose-dev.yaml restart web +``` + +## How video is transcoded + +Original files get uploaded to the application server, and they get stored there as FileFields. + +If files are videos and the duration is greater than a number (defined on settings, I think 4minutes), they are also broken in chunks, so one Encode object per chunk, for all enabled EncodeProfiles. + +Then the workers start picking Encode objects and they transcode the chunks, so if a chunk gets transcoded correctly, the original file (the small chunk) gets replaced by the transcoded file, and the Encode object status is marked as 'success'. + + +original.mp4 (1G, 720px)--> Encode1 (100MB, 240px, chunk=True), Encode2 (100MB, 240px, chunk=True)...EncodeXX (100MB, 720px, chunk=True) ---> when all Encode objects are success, for a resolution, they get concatenated to the original_resolution.mp4 file and this gets stored as Encode object (chunk=False). This is what is available for download. + +Apparently the Encode object is used to store Encoded files that are served eventually (chunk=False, status='success'), but also files while they are on their way to get transcoded (chunk=True, status='pending/etc') + +(Parenthesis opening) +there is also an experimental small service (not commited to the repo currently) that speaks only through API and a) gets tasks to run, b) returns results. So it makes a request and receives an ffmpeg command, plus a file, it runs the ffmpeg command, and returns the result.I've used this mechanism on a number of installations to migrate existing videos through more servers/cpu and has worked with only one problem, some temporary files needed to be removed from the servers (through a periodic task, not so big problem) +(Parenthesis closing) + + +When the Encode object is marked as success and chunk=False, and thus is available for download/stream, there is a task that gets started and saves an HLS version of the file (1 mp4-->x number of small .ts chunks). This would be FILES_C + +This mechanism allows for workers that have access on the same filesystem (either localhost, or through a shared network filesystem, eg NFS/EFS) to work on the same time and produce results. diff --git a/docs/images/CC-display.png b/docs/images/CC-display.png new file mode 100755 index 0000000..9f64d8f Binary files /dev/null and b/docs/images/CC-display.png differ diff --git a/docs/images/Click-ADD-button.png b/docs/images/Click-ADD-button.png new file mode 100755 index 0000000..df02315 Binary files /dev/null and b/docs/images/Click-ADD-button.png differ diff --git a/docs/images/Click-Browse-button.png b/docs/images/Click-Browse-button.png new file mode 100755 index 0000000..0172b5f Binary files /dev/null and b/docs/images/Click-Browse-button.png differ diff --git a/docs/images/Click-Download-Button.png b/docs/images/Click-Download-Button.png new file mode 100755 index 0000000..a483848 Binary files /dev/null and b/docs/images/Click-Download-Button.png differ diff --git a/docs/images/Click-EDIT-SUBTITLE.png b/docs/images/Click-EDIT-SUBTITLE.png new file mode 100755 index 0000000..33ef8ac Binary files /dev/null and b/docs/images/Click-EDIT-SUBTITLE.png differ diff --git a/docs/images/Click-Edit-Media-button.png b/docs/images/Click-Edit-Media-button.png new file mode 100755 index 0000000..c9b83db Binary files /dev/null and b/docs/images/Click-Edit-Media-button.png differ diff --git a/docs/images/Click-Subtitle-Language-Menu.png b/docs/images/Click-Subtitle-Language-Menu.png new file mode 100755 index 0000000..2533642 Binary files /dev/null and b/docs/images/Click-Subtitle-Language-Menu.png differ diff --git a/docs/images/Click-Upload-Media-button.png b/docs/images/Click-Upload-Media-button.png new file mode 100755 index 0000000..df90753 Binary files /dev/null and b/docs/images/Click-Upload-Media-button.png differ diff --git a/docs/images/Click-View-media-button.png b/docs/images/Click-View-media-button.png new file mode 100755 index 0000000..08e5b97 Binary files /dev/null and b/docs/images/Click-View-media-button.png differ diff --git a/docs/images/Click-version-download.png b/docs/images/Click-version-download.png new file mode 100755 index 0000000..dbea115 Binary files /dev/null and b/docs/images/Click-version-download.png differ diff --git a/docs/images/Continue-button.png b/docs/images/Continue-button.png new file mode 100755 index 0000000..9b857c7 Binary files /dev/null and b/docs/images/Continue-button.png differ diff --git a/docs/images/Edit-Media-Metadata-1.png b/docs/images/Edit-Media-Metadata-1.png new file mode 100755 index 0000000..f79ecf3 Binary files /dev/null and b/docs/images/Edit-Media-Metadata-1.png differ diff --git a/docs/images/Edit-Media-Metadata-2.png b/docs/images/Edit-Media-Metadata-2.png new file mode 100755 index 0000000..51d9d6f Binary files /dev/null and b/docs/images/Edit-Media-Metadata-2.png differ diff --git a/docs/images/Pause-button.png b/docs/images/Pause-button.png new file mode 100755 index 0000000..9eaae1d Binary files /dev/null and b/docs/images/Pause-button.png differ diff --git a/docs/images/Processing.png b/docs/images/Processing.png new file mode 100755 index 0000000..e055e61 Binary files /dev/null and b/docs/images/Processing.png differ diff --git a/docs/images/Save-File.png b/docs/images/Save-File.png new file mode 100755 index 0000000..a68a5be Binary files /dev/null and b/docs/images/Save-File.png differ diff --git a/docs/images/Select-Media-File-Click-Open.png b/docs/images/Select-Media-File-Click-Open.png new file mode 100755 index 0000000..f9c6bc7 Binary files /dev/null and b/docs/images/Select-Media-File-Click-Open.png differ diff --git a/docs/images/Subtitles-captions1.png b/docs/images/Subtitles-captions1.png new file mode 100755 index 0000000..aee45c8 Binary files /dev/null and b/docs/images/Subtitles-captions1.png differ diff --git a/docs/images/Subtitles-captions2.png b/docs/images/Subtitles-captions2.png new file mode 100755 index 0000000..9d30ede Binary files /dev/null and b/docs/images/Subtitles-captions2.png differ diff --git a/docs/images/Subtitles-captions3.png b/docs/images/Subtitles-captions3.png new file mode 100755 index 0000000..cc0c51d Binary files /dev/null and b/docs/images/Subtitles-captions3.png differ diff --git a/docs/images/Uploading.png b/docs/images/Uploading.png new file mode 100755 index 0000000..383bc4b Binary files /dev/null and b/docs/images/Uploading.png differ diff --git a/docs/user_docs.md b/docs/user_docs.md new file mode 100644 index 0000000..cfb3c37 --- /dev/null +++ b/docs/user_docs.md @@ -0,0 +1,208 @@ +# Users documentation + +## Table of contents +- [Uploading media](#uploading-media) +- [Downloading media](#downloading-media) +- [Adding captions/subtitles](#adding-captionssubtitles) +- [Search media](#search-media) +- [Share media](#share-media) +- [Embed media](#embed-media) +- [Customize my profile options](#customize-my-profile-options) + +## Uploading media + +### How to Upload Media + +Uploading media is as simple as clicking the _Upload Media_ button, waiting for media to upload, and then clicking the media to add metadata (title, description etc.) by filling out a form. + +#### Click Upload Button + +Click the _Upload Media_ button from the right-side of the screen at the top: + +
+ +
+ +#### Upload Page + +Clicking the _Upload Media_ button takes you to the upload page at a URL like this: + +https://demo.mediacms.io/upload + +#### Click Browse Button + +Here you should click the _"Browse your files"_ button (or drag and drop a file from your desktop): + ++ +
+ +#### Select media file and click Open button + +Select the media file that you want to upload and click the _"Open"_ button: + ++ +
+ +#### Wait for file to upload + +Wait for the file to finish uploading: + ++ +
+ +#### Pause uploading + +If you want you can pause upload by clicking _Pause button_: + ++ +
+ +#### Continue uploading + +Continue upload by clicking _Continue button_: + ++ +
+ +#### Wait for media to finish Processing + +Wait for the media file to finish Processing: + ++ +
+ +#### Click View media button + +Click the View media button to open the media page: + ++ +
+ +#### Media will be in the encoding queue + +The media will take some time to finish encoding (MediaCMS will transcode the file to several formats and resolutions). Meanwhile you can edit the media file to add metadata. + +#### Click Edit Media button + +Click the EDIT MEDIA button to add metadata and configure the poster image: + ++ +
+ +#### Add Metadata (Summary, Description etc.) + +Make sure you fill in all the required fields, and try to complete as many of the non-required fields as possible. This ensures the database is populated with useful meta-data to help others access useful information about you and your video. + ++ +
+ ++ +
+ + + +## Downloading media + +MediaCMS offers a configurable option whereby users can make their media files available for download. Downloads are available for transcoded files, and the original file. + +#### How To Enable Download + +Visit the media view page and choose the EDIT MEDIA button. + +Select the checkbox for "Allow Downloads" + +#### How To Download Media + +Visit the media view page and click the DOWNLOAD button below the video player. + ++ +
+ +Choose the version you wish to download - a transcoded file or the original file: + ++ +
+ +Choose Save File and click the OK button. + ++ +
+ + + +## Adding captions/subtitles + +With MediaCMS you can add subtitles/captions to your video by uploading a subtitles file in the popular .vtt format. + +### Visit Media Page & Click EDIT SUBTITLE Button + +Visit the "single media page" for the media you wish to add subtitles/captions to and click the EDIT SUBTITLES button: + ++ +
+ +### Upload Subtitles in .vtt Format + +Click the Language menu to select the correct language for your Subtitles/Captions file: + ++ +
+ +Choose the correct Language for your file: + ++ +
+ +Click Browse to find a subtitles/captions file on your computer (if your file is not in the .vtt format, you may find a conversion service on the Internet): + ++ +
+ +Choose a .vtt subtitles/captions file from your computer: + ++ +
+ +Click the Add button to add the file to your media: + ++ +
+ +### View Subtitles/Captions in Video Player + +You can now watch the captions/subtitles play back in the video player - and toggle display on/off by clicking the CC button: + ++ +
+ +## Search media +How search can be used + +## Share media +How to share media + +## Embed media +How to use the embed media option + +## Customize my profile options +Customize profile and channel diff --git a/install.sh b/install.sh index 402d2ed..2fb9a34 100644 --- a/install.sh +++ b/install.sh @@ -21,21 +21,24 @@ It is expected to run on a new system **with no running instances of any these s done -if [[ `lsb_release -d` == *"Ubuntu 20"* ]]; then +osVersion=$(lsb_release -d) +if [[ $osVersion == *"Ubuntu 20"* ]] || [[ $osVersion == *"Ubuntu 18"* ]] || [[ $osVersion == *"buster"* ]]; then echo 'Performing system update and dependency installation, this will take a few minutes' - apt-get update && apt-get -y upgrade && apt install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip ffmpeg imagemagick python3-certbot-nginx certbot wget -y -elif [[ `lsb_release -d` = *"Ubuntu 18"* ]]; then - echo 'Performing system update and dependency installation, this will take a few minutes' - apt-get update && apt-get -y upgrade && apt install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip ffmpeg imagemagick python3-certbot-nginx certbot wget -y -# added check for Debian 10 (buster) -elif [[ `lsb_release -d` == *"buster"* ]]; then - echo 'Performing system update and dependency installation, this will take a few minutes' - apt-get update && apt-get -y upgrade && apt install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip ffmpeg imagemagick python3-certbot-nginx certbot wget -y + apt-get update && apt-get -y upgrade && apt-get install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip imagemagick python3-certbot-nginx certbot wget xz-utils -y else echo "This script is tested for Ubuntu 18 and 20 versions only, if you want to try MediaCMS on another system you have to perform the manual installation" -exit + exit fi +# install ffmpeg +echo "Downloading and installing ffmpeg" +wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz +mkdir -p tmp +tar -xf ffmpeg-release-amd64-static.tar.xz --strip-components 1 -C tmp +cp -v tmp/{ffmpeg,ffprobe,qt-faststart} /usr/local/bin +rm -rf tmp ffmpeg-release-amd64-static.tar.xz +echo "ffmpeg installed to /usr/local/bin" + read -p "Enter portal URL, or press enter for localhost : " FRONTEND_HOST read -p "Enter portal name, or press enter for 'MediaCMS : " PORTAL_NAME