diff --git a/newsfragments/fix-build-ssh-path-to-be-relative.bugfix b/newsfragments/fix-build-ssh-path-to-be-relative.bugfix
new file mode 100644
index 0000000..4cff5f4
--- /dev/null
+++ b/newsfragments/fix-build-ssh-path-to-be-relative.bugfix
@@ -0,0 +1 @@
+Fixed build ssh path to a local SSH key, to be relative to the directory of compose file.
diff --git a/podman_compose.py b/podman_compose.py
index f0841d5..0bf19a6 100755
--- a/podman_compose.py
+++ b/podman_compose.py
@@ -2644,6 +2644,16 @@ def is_path_git_url(path):
     return r.scheme == 'git' or r.path.endswith('.git')
 
 
+def adjust_build_ssh_key_paths(compose, agent_or_key):
+    # when using a custom id for ssh property, path to a local SSH key is provided after "="
+    parts = agent_or_key.split("=", 1)
+    if len(parts) == 1:
+        return agent_or_key
+    name, path = parts
+    path = os.path.expanduser(path)
+    return name + "=" + os.path.join(compose.dirname, path)
+
+
 def container_to_build_args(compose, cnt, args, path_exists, cleanup_callbacks=None):
     build_desc = cnt["build"]
     if not hasattr(build_desc, "items"):
@@ -2712,6 +2722,7 @@ def container_to_build_args(compose, cnt, args, path_exists, cleanup_callbacks=N
     if "target" in build_desc:
         build_args.extend(["--target", build_desc["target"]])
     for agent_or_key in norm_as_list(build_desc.get("ssh", {})):
+        agent_or_key = adjust_build_ssh_key_paths(compose, agent_or_key)
         build_args.extend(["--ssh", agent_or_key])
     container_to_ulimit_build_args(cnt, build_args)
     if getattr(args, "no_cache", None):
diff --git a/test-requirements.txt b/test-requirements.txt
index cd6c533..e53f1d0 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,5 +1,6 @@
 -e .
 coverage==7.4.3
+cryptography==44.0.3
 parameterized==0.9.0
 pytest==8.0.2
 tox==4.13.0
diff --git a/tests/integration/build_ssh/__init__.py b/tests/integration/build_ssh/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/tests/integration/build_ssh/__init__.py
@@ -0,0 +1 @@
+
diff --git a/tests/unit/test_container_to_build_args.py b/tests/unit/test_container_to_build_args.py
index cd52aca..3b079ea 100644
--- a/tests/unit/test_container_to_build_args.py
+++ b/tests/unit/test_container_to_build_args.py
@@ -219,3 +219,146 @@ class TestContainerToBuildArgs(unittest.TestCase):
 
         with self.assertRaises(OSError):
             container_to_build_args(c, cnt, args, lambda path: False)
+
+    def test_build_ssh_absolute_path(self):
+        c = create_compose_mock()
+
+        cnt = get_minimal_container()
+        cnt['build']['ssh'] = ["id1=/test1"]
+        args = get_minimal_args()
+
+        args = container_to_build_args(c, cnt, args, lambda path: True)
+        self.assertEqual(
+            args,
+            [
+                '-f',
+                'Containerfile',
+                '-t',
+                'new-image',
+                '--ssh',
+                'id1=/test1',
+                '--no-cache',
+                '--pull-always',
+                '.',
+            ],
+        )
+
+    def test_build_ssh_relative_path(self):
+        c = create_compose_mock()
+
+        cnt = get_minimal_container()
+        cnt['build']['ssh'] = ["id1=id1/test1"]
+        args = get_minimal_args()
+
+        args = container_to_build_args(c, cnt, args, lambda path: True)
+        self.assertEqual(
+            args,
+            [
+                '-f',
+                'Containerfile',
+                '-t',
+                'new-image',
+                '--ssh',
+                'id1=test_dirname/id1/test1',
+                '--no-cache',
+                '--pull-always',
+                '.',
+            ],
+        )
+
+    def test_build_ssh_working_dir(self):
+        c = create_compose_mock()
+
+        cnt = get_minimal_container()
+        cnt['build']['ssh'] = ["id1=./test1"]
+        args = get_minimal_args()
+
+        args = container_to_build_args(c, cnt, args, lambda path: True)
+        self.assertEqual(
+            args,
+            [
+                '-f',
+                'Containerfile',
+                '-t',
+                'new-image',
+                '--ssh',
+                'id1=test_dirname/./test1',
+                '--no-cache',
+                '--pull-always',
+                '.',
+            ],
+        )
+
+    @mock.patch.dict(os.environ, {"HOME": "/home/user"}, clear=True)
+    def test_build_ssh_path_home_dir(self):
+        c = create_compose_mock()
+
+        cnt = get_minimal_container()
+        cnt['build']['ssh'] = ["id1=~/test1"]
+        args = get_minimal_args()
+
+        args = container_to_build_args(c, cnt, args, lambda path: True)
+        self.assertEqual(
+            args,
+            [
+                '-f',
+                'Containerfile',
+                '-t',
+                'new-image',
+                '--ssh',
+                'id1=/home/user/test1',
+                '--no-cache',
+                '--pull-always',
+                '.',
+            ],
+        )
+
+    def test_build_ssh_map(self):
+        c = create_compose_mock()
+
+        cnt = get_minimal_container()
+        cnt['build']['ssh'] = {"id1": "test1", "id2": "test2"}
+        args = get_minimal_args()
+
+        args = container_to_build_args(c, cnt, args, lambda path: True)
+        self.assertEqual(
+            args,
+            [
+                '-f',
+                'Containerfile',
+                '-t',
+                'new-image',
+                '--ssh',
+                'id1=test_dirname/test1',
+                '--ssh',
+                'id2=test_dirname/test2',
+                '--no-cache',
+                '--pull-always',
+                '.',
+            ],
+        )
+
+    def test_build_ssh_array(self):
+        c = create_compose_mock()
+
+        cnt = get_minimal_container()
+        cnt['build']['ssh'] = ['id1=test1', 'id2=test2']
+        args = get_minimal_args()
+
+        args = container_to_build_args(c, cnt, args, lambda path: True)
+        self.assertEqual(
+            args,
+            [
+                '-f',
+                'Containerfile',
+                '-t',
+                'new-image',
+                '--ssh',
+                'id1=test_dirname/test1',
+                '--ssh',
+                'id2=test_dirname/test2',
+                '--no-cache',
+                '--pull-always',
+                '.',
+            ],
+        )