Merge pull request #1279 from mokibit/add-volume-bind-create-host-path-option

Add volume bind `create_host_path` option

RELEASING.md

@@ -45,3 +45,19 @@ Step 5: Create a release on Github
 
 The release notes must be added manually by drafting a release on the GitHub UI at
 https://github.com/containers/podman-compose/releases.
+
+Step 6: Upload to Pypi
+----------------------
+
+Run the following command to build artifacts:
+
+```
+rm -rf build dist
+python3 -m build
+```
+
+Upload to Pypi
+
+```
+python3 -m twine upload dist/*
+```

newsfragments/add-volumes-bind-create-host-path-option.bugfix

@@ -0,0 +1 @@
+Implemented volumes bind `create_host_path` option.

newsfragments/compose_profiles_env.feature

@@ -0,0 +1 @@
+Support COMPOSE_PROFILES environment variable to enable profiles.

podman_compose.py

@@ -405,6 +405,12 @@ async def assert_volume(compose: PodmanCompose, mount_dict: dict[str, Any]) -> N
         mount_src = mount_dict["source"]
         mount_src = os.path.realpath(os.path.join(basedir, os.path.expanduser(mount_src)))
         if not os.path.exists(mount_src):
+            bind_opts = mount_dict.get("bind", {})
+            if "create_host_path" in bind_opts and not bind_opts["create_host_path"]:
+                raise ValueError(
+                    "invalid mount config for type 'bind': bind source path does not exist: "
+                    f"{mount_src}"
+                )
             try:
                 os.makedirs(mount_src, exist_ok=True)
             except OSError:
@@ -2231,6 +2237,11 @@ class PodmanCompose:
             env_vars = norm_as_dict(args.env)
             self.environ.update(env_vars)  # type: ignore[arg-type]
 
+        profiles_from_env = {
+            p.strip() for p in self.environ.get("COMPOSE_PROFILES", "").split(",") if p.strip()
+        }
+        requested_profiles = set(args.profile).union(profiles_from_env)
+
         compose: dict[str, Any] = {}
         # Iterate over files primitively to allow appending to files in-loop
         files_iter = iter(files)
@@ -2296,7 +2307,7 @@ class PodmanCompose:
                 # Solution is to remove 'include' key from compose obj. This doesn't break
                 # having `include` present and correctly processed in included files
                 del compose["include"]
-        resolved_services = self._resolve_profiles(compose.get("services", {}), set(args.profile))
+        resolved_services = self._resolve_profiles(compose.get("services", {}), requested_profiles)
         compose["services"] = resolved_services
         if not getattr(args, "no_normalize", None):
             compose = normalize_final(compose, self.dirname)
@@ -2318,7 +2329,7 @@ class PodmanCompose:
             services = {}
             log.warning("WARNING: No services defined")
         # include services with no profile defined or the selected profiles
-        services = self._resolve_profiles(services, set(args.profile))
+        services = self._resolve_profiles(services, requested_profiles)
 
         # NOTE: maybe add "extends.service" to _deps at this stage
         flat_deps(services, with_extends=True)

tests/integration/profile/test_podman_compose_up_down.py

@@ -90,3 +90,71 @@ class TestUpDown(unittest.TestCase, RunSubprocessMixin):
             actual_services[service] = service in actual_output
 
         self.assertEqual(expected_services, actual_services)
+
+    @parameterized.expand(
+        [
+            (
+                ["--profile", "profile-1"],
+                "profile-2",
+                {"default-service": True, "service-1": True, "service-2": True},
+            ),
+            (
+                [],
+                "profile-1,profile-2",
+                {"default-service": True, "service-1": True, "service-2": True},
+            ),
+            (
+                [],
+                "profile-1, profile-2",
+                {"default-service": True, "service-1": True, "service-2": True},
+            ),
+            (
+                [],
+                "",
+                {"default-service": True, "service-1": False, "service-2": False},
+            ),
+            (
+                [],
+                ",",
+                {"default-service": True, "service-1": False, "service-2": False},
+            ),
+        ],
+    )
+    def test_up_with_compose_profiles_env(
+        self, profiles: List[str], compose_profiles: str, expected_services: dict
+    ) -> None:
+        """
+        Tests the `up` command when the `COMPOSE_PROFILES` environment variable is set.
+        """
+        up_cmd = [
+            "coverage",
+            "run",
+            podman_compose_path(),
+            "-f",
+            profile_compose_file(),
+        ]
+        up_cmd.extend(profiles)
+        up_cmd.extend(["up", "-d"])
+
+        env = os.environ.copy()
+        env["COMPOSE_PROFILES"] = compose_profiles
+
+        self.run_subprocess_assert_returncode(up_cmd, env=env)
+
+        check_cmd = [
+            "podman",
+            "container",
+            "ps",
+            "--format",
+            '"{{.Names}}"',
+        ]
+        out, _ = self.run_subprocess_assert_returncode(check_cmd)
+
+        self.assertEqual(len(expected_services), 3)
+        actual_output = out.decode("utf-8")
+
+        actual_services = {}
+        for service, _ in expected_services.items():
+            actual_services[service] = service in actual_output
+
+        self.assertEqual(expected_services, actual_services)

tests/integration/vol/long_syntax/docker-compose.yml

@@ -0,0 +1,27 @@
+version: "3"
+services:
+  create_host_path_default_true: # default is to always create
+    image: nopush/podman-compose-test
+    command: ["/bin/busybox", "mkdir", "cont/test_dir/new_dir"]
+    volumes:
+      - type: bind
+        source: ./test_dir
+        target: /cont/test_dir
+  create_host_path_true:
+    image: nopush/podman-compose-test
+    command: ["/bin/busybox", "mkdir", "cont/test_dir/new_dir"]
+    volumes:
+      - type: bind
+        source: ./test_dir
+        target: /cont/test_dir
+        bind:
+          create_host_path: true
+  create_host_path_false:
+    image: nopush/podman-compose-test
+    command: ["/bin/busybox", "mkdir", "cont/test_dir/new_dir"]
+    volumes:
+      - type: bind
+        source: ./test_dir
+        target: /cont/test_dir
+        bind:
+          create_host_path: false

tests/integration/vol/long_syntax/test_podman_compose_vol_long_syntax.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: GPL-2.0
+
+import os
+import shutil
+import unittest
+
+from parameterized import parameterized
+
+from tests.integration.test_utils import RunSubprocessMixin
+from tests.integration.test_utils import podman_compose_path
+from tests.integration.test_utils import test_path
+
+
+def compose_yaml_path(test_ref_folder: str) -> str:
+    return os.path.join(test_path(), "vol", test_ref_folder, "docker-compose.yml")
+
+
+class TestComposeVolLongSyntax(unittest.TestCase, RunSubprocessMixin):
+    @parameterized.expand([
+        (True, "create_host_path_default_true"),
+        (True, "create_host_path_true"),
+        (True, "create_host_path_false"),
+        (False, "create_host_path_default_true"),
+        (False, "create_host_path_true"),
+    ])
+    def test_source_host_dir(self, source_dir_exists: bool, service_name: str) -> None:
+        project_dir = os.path.join(test_path(), "vol", "long_syntax")
+        if source_dir_exists:
+            # create host source directory for volume
+            os.mkdir(os.path.join(project_dir, "test_dir"))
+        else:
+            # make sure there is no such directory
+            self.assertFalse(os.path.isdir(os.path.join(project_dir, "test_dir")))
+
+        try:
+            self.run_subprocess_assert_returncode(
+                [
+                    podman_compose_path(),
+                    "-f",
+                    compose_yaml_path("long_syntax"),
+                    "up",
+                    "-d",
+                    f"{service_name}",
+                ],
+                0,
+            )
+            # command of the service creates a new directory on mounted directory 'test_dir' in
+            # the container. Check if host directory now has the same directory. It represents a
+            # successful mount.
+            # If source host directory does not exist, it is created
+            self.assertTrue(os.path.isdir(os.path.join(project_dir, "test_dir/new_dir")))
+
+        finally:
+            self.run_subprocess_assert_returncode([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path("long_syntax"),
+                "down",
+                "-t",
+                "0",
+            ])
+            shutil.rmtree(os.path.join(project_dir, "test_dir"))
+
+    def test_no_host_source_dir_create_host_path_false(self) -> None:
+        try:
+            _, error = self.run_subprocess_assert_returncode(
+                [
+                    podman_compose_path(),
+                    "-f",
+                    compose_yaml_path("long_syntax"),
+                    "up",
+                    "-d",
+                    "create_host_path_false",
+                ],
+                1,
+            )
+            self.assertIn(
+                b"invalid mount config for type 'bind': bind source path does not exist:", error
+            )
+        finally:
+            self.run_subprocess([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path("long_syntax"),
+                "down",
+                "-t",
+                "0",
+            ])

tests/integration/vol/short_syntax/docker-compose.yml

@@ -0,0 +1,13 @@
+version: "3"
+services:
+  test1:
+    image: nopush/podman-compose-test
+    #["/bin/busybox", "sh", "-c", "env | grep ZZVAR3"]
+    command: ["/bin/busybox", "mkdir", "cont/test_dir/new_dir"]
+    volumes:
+      - ./test_dir:/cont/test_dir
+  test2:
+    image: nopush/podman-compose-test
+    command: ["/bin/busybox", "mkdir", "cont/test_dir/new_dir"]
+    volumes:
+      - ./test_dir:/cont/test_dir

tests/integration/vol/short_syntax/test_podman_compose_vol_short_syntax.py

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: GPL-2.0
+
+import os
+import shutil
+import unittest
+
+from parameterized import parameterized
+
+from tests.integration.test_utils import RunSubprocessMixin
+from tests.integration.test_utils import podman_compose_path
+from tests.integration.test_utils import test_path
+
+
+def compose_yaml_path(test_ref_folder: str) -> str:
+    return os.path.join(test_path(), "vol", test_ref_folder, "docker-compose.yml")
+
+
+class TestComposeVolShortSyntax(unittest.TestCase, RunSubprocessMixin):
+    @parameterized.expand([
+        (True, "test1"),
+        (False, "test2"),
+    ])
+    def test_source_host_dir(self, source_host_dir_exists: bool, service_name: str) -> None:
+        project_dir = os.path.join(test_path(), "vol", "short_syntax")
+        # create host source directory for volume
+        if source_host_dir_exists:
+            os.mkdir(os.path.join(project_dir, "test_dir"))
+        try:
+            self.run_subprocess_assert_returncode(
+                [
+                    podman_compose_path(),
+                    "-f",
+                    compose_yaml_path("short_syntax"),
+                    "up",
+                    "-d",
+                    f"{service_name}",
+                ],
+                0,
+            )
+            # command of the service creates a new directory on mounted directory 'test_dir' in
+            # container. Check if host directory now has the same directory. It represents a
+            # successful mount.
+            # On service test2 source host directory is created as it did not exist
+            self.assertTrue(os.path.isdir(os.path.join(project_dir, "test_dir/new_dir")))
+
+        finally:
+            self.run_subprocess([
+                podman_compose_path(),
+                "-f",
+                compose_yaml_path("short_syntax"),
+                "down",
+                "-t",
+                "0",
+            ])
+            shutil.rmtree(os.path.join(project_dir, "test_dir"))

tests/unit/test_container_to_args.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 
 import os
+import shutil
 import unittest
 from typing import Any
 from unittest import mock
@@ -679,6 +680,71 @@ class TestContainerToArgs(unittest.IsolatedAsyncioTestCase):
             ],
         )
 
+    @parameterized.expand([
+        (
+            "create_host_path_set_to_true",
+            {"bind": {"create_host_path": True}},
+        ),
+        (
+            "create_host_path_default_true",
+            {},
+        ),
+    ])
+    async def test_volumes_bind_mount_create_source_dir(self, test_name: str, bind: dict) -> None:
+        # creates a missing source dir
+        c = create_compose_mock()
+        c.prefer_volume_over_mount = True
+        cnt = get_minimal_container()
+
+        cnt["_service"] = cnt["service_name"]
+
+        volume_info = {
+            "type": "bind",
+            "source": "./not_exists/foo",
+            "target": "/mnt",
+        }
+        volume_info.update(bind)
+        cnt["volumes"] = [
+            volume_info,
+        ]
+
+        args = await container_to_args(c, cnt)
+
+        self.assertEqual(
+            args,
+            [
+                "--name=project_name_service_name1",
+                "-d",
+                "-v",
+                f"{get_test_file_path('./test_dirname/not_exists/foo')}:/mnt",
+                "--network=bridge:alias=service_name",
+                "busybox",
+            ],
+        )
+        dir_path = get_test_file_path('./test_dirname/not_exists/foo')
+        shutil.rmtree(dir_path)
+
+    # throws an error as the source path does not exist and its creation was suppressed with the
+    # create_host_path = False option
+    async def test_volumes_bind_mount_source_does_not_exist(self) -> None:
+        c = create_compose_mock()
+        c.prefer_volume_over_mount = True
+        cnt = get_minimal_container()
+
+        cnt["_service"] = cnt["service_name"]
+
+        cnt["volumes"] = [
+            {
+                "type": "bind",
+                "source": "./not_exists/foo",
+                "target": "/mnt",
+                "bind": {"create_host_path": False},
+            }
+        ]
+
+        with self.assertRaises(ValueError):
+            await container_to_args(c, cnt)
+
     @parameterized.expand([
         ("not_compat", False, "test_project_name", "test_project_name_network1"),
         ("compat_no_dash", True, "test_project_name", "test_project_name_network1"),