pass volumes using -v

podman_compose.py

@@ -34,7 +34,7 @@ except ImportError:
 import json
 import yaml
 
-__version__ = '0.1.6dev'
+__version__ = '0.1.7dev'
 
 PY3 = sys.version_info[0] == 3
 if PY3:
@@ -46,7 +46,7 @@ is_str  = lambda s: isinstance(s, basestring)
 is_dict = lambda d: isinstance(d, dict)
 is_list = lambda l: not is_str(l) and not is_dict(l) and hasattr(l, "__iter__")
 # identity filter
-filteri = lambda a: filter(lambda i:i, a)
+filteri = lambda a: filter(lambda i: i, a)
 
 def try_int(i, fallback=None):
     try:
@@ -58,27 +58,24 @@ def try_int(i, fallback=None):
     return fallback
 
 dir_re = re.compile("^[~/\.]")
-propagation_re=re.compile("^(?:z|Z|r?shared|r?slave|r?private)$")
+propagation_re = re.compile("^(?:z|Z|r?shared|r?slave|r?private)$")
-
-# NOTE: if a named volume is used but not defined it gives
-# ERROR: Named volume "so and so" is used in service "xyz" but no declaration was found in the volumes section.
-# unless it's anon-volume
 
 def parse_short_mount(mount_str, basedir):
     mount_a = mount_str.split(':')
     mount_opt_dict = {}
     mount_opt = None
-    if len(mount_a)==1:
+    if len(mount_a) == 1:
-        # Just specify a path and let the Engine create a volume
+        # Anonymous: Just specify a path and let the engine creates the volume
         # - /var/lib/mysql
-        mount_src, mount_dst=None, mount_str
+        mount_src, mount_dst = None, mount_str
-    elif len(mount_a)==2:
+    elif len(mount_a) == 2:
         mount_src, mount_dst = mount_a
-        # dest must start with /, otherwise it's option
+        # dest must start with / like /foo:/var/lib/mysql
+        # otherwise it's option like /var/lib/mysql:rw
         if not mount_dst.startswith('/'):
             mount_dst, mount_opt = mount_a
             mount_src = None
-    elif len(mount_a)==3:
+    elif len(mount_a) == 3:
         mount_src, mount_dst, mount_opt = mount_a
     else:
         raise ValueError("could not parse mount "+mount_str)
@@ -98,24 +95,31 @@ def parse_short_mount(mount_str, basedir):
         mount_type = "volume"
     mount_opts = filteri((mount_opt or '').split(','))
     for opt in mount_opts:
-        if opt=='ro': mount_opt_dict["read_only"]=True
+        if opt == 'ro': mount_opt_dict["read_only"] = True
-        elif opt=='rw': mount_opt_dict["read_only"]=False
+        elif opt == 'rw': mount_opt_dict["read_only"] = False
-        elif opt=='delegated': mount_opt_dict["delegated"]=dict(propagation=opt)
+        elif opt in ('consistent', 'delegated', 'cached'):
-        elif opt=='cached': mount_opt_dict["cached"]=dict(propagation=opt)
+            mount_opt_dict["consistency"] = opt
-        elif propagation_re.match(opt): mount_opt_dict["bind"]=dict(propagation=opt)
+        elif propagation_re.match(opt): mount_opt_dict["bind"] = dict(propagation=opt)
         else:
             # TODO: ignore
             raise ValueError("unknown mount option "+opt)
     return dict(type=mount_type, source=mount_src, target=mount_dst, **mount_opt_dict)
 
+# NOTE: if a named volume is used but not defined it
+# gives ERROR: Named volume "abc" is used in service "xyz"
+#   but no declaration was found in the volumes section.
+# unless it's anonymous-volume
+
 def fix_mount_dict(mount_dict, proj_name, srv_name):
     """
     in-place fix mount dictionary to:
     - add missing source
     - prefix source with proj_name
     """
-    if mount_dict["type"]=="volume":
+    # if already applied nothing todo
-        source = mount_dict.get("source")
+    if "_source" in mount_dict: return mount_dict
+    if mount_dict["type"] == "volume":
+        source = mount_dict.get("source", None)
         # keep old source
         mount_dict["_source"] = source
         if not source:
@@ -152,7 +156,7 @@ def dicts_get(dicts, key, fallback='', fallback_empty=False):
     """
     value = None
     for d in dicts:
-        value = d.get(key)
+        value = d.get(key, None)
         if value is not None: break
     if not value:
         if fallback_empty or value is None:
@@ -218,8 +222,8 @@ def norm_ulimit(inner_value):
     if is_dict(inner_value):
         if not inner_value.keys() & {"soft", "hard"}:
             raise ValueError("expected at least one soft or hard limit")
-        soft = inner_value.get("soft", inner_value.get("hard"))
+        soft = inner_value.get("soft", inner_value.get("hard", None))
-        hard = inner_value.get("hard", inner_value.get("soft"))
+        hard = inner_value.get("hard", inner_value.get("soft", None))
         return "{}:{}".format(soft, hard)
     elif is_list(inner_value): return norm_ulimit(norm_as_dict(inner_value))
     # if int or string return as is
@@ -252,9 +256,9 @@ def move_list(dst, containers, key):
     """
     move key (like port forwarding) from containers to dst (a pod or a infra container)
     """
-    a = set(dst.get(key) or [])
+    a = set(dst.get(key, None) or [])
     for cnt in containers:
-        a0 = cnt.get(key)
+        a0 = cnt.get(key, None)
         if a0:
             a.update(a0)
             del cnt[key]
@@ -326,7 +330,7 @@ def tr_cntnet(project_name, services, given_containers):
     )
     for cnt0 in given_containers:
         cnt = dict(cnt0, network_mode="container:"+infra_name)
-        deps = cnt.get("depends_on") or []
+        deps = cnt.get("depends_on", None) or []
         deps.append(infra_name)
         cnt["depends_on"] = deps
         # adjust hosts to point to localhost, TODO: adjust host env
@@ -366,73 +370,60 @@ def tr_1podfw(project_name, services, given_containers):
     return pods, containers
 
 
-def mount_dict_vol_to_bind(compose, mount_dict):
+def assert_volume(compose, mount_dict):
     """
     inspect volume to get directory
     create volume if needed
-    and return mount_dict as bind of that directory
     """
+    if mount_dict["type"] != "volume": return
     proj_name = compose.project_name
     shared_vols = compose.shared_vols
-    if mount_dict["type"]!="volume": return mount_dict
+
     vol_name_orig = mount_dict.get("_source", None)
     vol_name = mount_dict["source"]
     print("podman volume inspect {vol_name} || podman volume create {vol_name}".format(vol_name=vol_name))
+    # TODO: might move to using "volume list"
     # podman volume list --format '{{.Name}}\t{{.MountPoint}}' -f 'label=io.podman.compose.project=HERE'
     try: out = compose.podman.output(["volume", "inspect", vol_name]).decode('utf-8')
     except subprocess.CalledProcessError:
         compose.podman.output(["volume", "create", "--label", "io.podman.compose.project={}".format(proj_name), vol_name])
         out = compose.podman.output(["volume", "inspect", vol_name]).decode('utf-8')
-    try:
-        src = json.loads(out)[0]["mountPoint"]
-    except KeyError:
-        src = json.loads(out)[0]["Mountpoint"]
-    ret=dict(mount_dict, type="bind", source=src, _vol=vol_name)
-    bind_prop=ret.get("bind", {}).get("propagation")
-    if not bind_prop:
-        if "bind" not in ret:
-            ret["bind"]={}
-        # if in top level volumes then it's shared bind-propagation=z
-        if vol_name_orig and vol_name_orig in shared_vols:
-            ret["bind"]["propagation"]="z"
-        else:
-            ret["bind"]["propagation"]="Z"
-    try: del ret["volume"]
-    except KeyError: pass
-    return ret
 
-def mount_desc_to_args(compose, mount_desc, srv_name, cnt_name):
+def mount_desc_to_mount_args(compose, mount_desc, srv_name, cnt_name):
     basedir = compose.dirname
     proj_name = compose.project_name
     shared_vols = compose.shared_vols
-    if is_str(mount_desc): mount_desc=parse_short_mount(mount_desc, basedir)
+    mount_type = mount_desc.get("type", None)
-    # not needed
+    source = mount_desc.get("source", None)
-    # podman support: podman run --rm -ti --mount type=volume,source=myvol,destination=/delme busybox
-    mount_desc = mount_dict_vol_to_bind(compose, fix_mount_dict(mount_desc, proj_name, srv_name))
-    mount_type = mount_desc.get("type")
-    source = mount_desc.get("source")
     target = mount_desc["target"]
-    opts=[]
+    opts = []
-    if mount_desc.get("bind"):
+    if mount_desc.get(mount_type, None):
-        bind_prop=mount_desc["bind"].get("propagation")
+        # TODO: we might need to add mount_dict[mount_type]["propagation"] = "z"
-        if bind_prop: opts.append("bind-propagation={}".format(bind_prop))
+        mount_prop = mount_desc.get(mount_type, {}).get("propagation", None)
+        if mount_prop: opts.append("{}-propagation={}".format(mount_type, mount_prop))
     if mount_desc.get("read_only", False): opts.append("ro")
-    if mount_type=='tmpfs':
+    if mount_type == 'tmpfs':
         tmpfs_opts = mount_desc.get("tmpfs", {})
-        tmpfs_size = tmpfs_opts.get("size")
+        tmpfs_size = tmpfs_opts.get("size", None)
         if tmpfs_size:
             opts.append("tmpfs-size={}".format(tmpfs_size))
-        tmpfs_mode = tmpfs_opts.get("mode")
+        tmpfs_mode = tmpfs_opts.get("mode", None)
         if tmpfs_mode:
             opts.append("tmpfs-mode={}".format(tmpfs_mode))
-    opts=",".join(opts)
+    opts = ",".join(opts)
-    if mount_type=='bind':
+    if mount_type == 'bind':
         return "type=bind,source={source},destination={target},{opts}".format(
             source=source,
             target=target,
             opts=opts
         ).rstrip(",")
-    elif mount_type=='tmpfs':
+    elif mount_type == 'volume':
+        return "type=volume,source={source},destination={target},{opts}".format(
+            source=source,
+            target=target,
+            opts=opts
+        ).rstrip(",")
+    elif mount_type == 'tmpfs':
         return "type=tmpfs,destination={target},{opts}".format(
             target=target,
             opts=opts
@@ -440,8 +431,6 @@ def mount_desc_to_args(compose, mount_desc, srv_name, cnt_name):
     else:
         raise ValueError("unknown mount type:"+mount_type)
 
-
-
 def container_to_ulimit_args(cnt, podman_args):
     ulimit = cnt.get('ulimits', [])
     if ulimit is not None:
@@ -455,16 +444,69 @@ def container_to_ulimit_args(cnt, podman_args):
             for i in ulimit:
                 podman_args.extend(['--ulimit', i])
 
+def mount_desc_to_volume_args(compose, mount_desc, srv_name, cnt_name):
+    basedir = compose.dirname
+    proj_name = compose.project_name
+    shared_vols = compose.shared_vols
+    mount_type = mount_desc["type"]
+    source = mount_desc.get("source", None)
+    target = mount_desc["target"]
+    opts = []
+    if mount_type != 'bind' and mount_type != 'volume':
+        raise ValueError("unknown mount type:"+mount_type)
+    propagations = set(filteri(mount_desc.get(mount_type, {}).get("propagation", "").split(',')))
+    if mount_type != 'bind':
+        propagations.update(filteri(mount_desc.get('bind', {}).get("propagation", "").split(',')))
+    opts.extend(propagations)
+    # --volume, -v[=[[SOURCE-VOLUME|HOST-DIR:]CONTAINER-DIR[:OPTIONS]]]
+    # [rw|ro]
+    # [z|Z]
+    # [[r]shared|[r]slave|[r]private]
+    # [[r]bind]
+    # [noexec|exec]
+    # [nodev|dev]
+    # [nosuid|suid]
+    read_only = mount_desc.get("read_only", None)
+    if read_only is not None:
+        opts.append('ro' if read_only else 'rw')
+    args = f'{source}:{target}'
+    if opts: args += ':' + ','.join(opts)
+    return args
 
+def get_mount_args(compose, cnt, volume):
+    proj_name = compose.project_name
+    srv_name = cnt['_service']
+    basedir = compose.dirname
+    if is_str(volume): volume = parse_short_mount(volume, basedir)
+    mount_type = volume["type"]
+    assert_volume(compose, fix_mount_dict(volume, proj_name, srv_name))
+    if compose._prefer_volume_over_mount:
+        if mount_type == 'tmpfs':
+            # TODO: --tmpfs /tmp:rw,size=787448k,mode=1777
+            args = volume['target']
+            tmpfs_opts = volume.get("tmpfs", {})
+            opts = []
+            size = tmpfs_opts.get("size", None)
+            if size: opts.append('size={}'.format(size))
+            mode = tmpfs_opts.get("mode", None)
+            if mode: opts.append('mode={}'.format(size))
+            if opts: args += ':' + ','.join(opts)
+            return ['--tmpfs', args]
+        else:
+            args = mount_desc_to_volume_args(compose, volume, srv_name, cnt['name'])
+            return ['-v', args]
+    else:
+        args = mount_desc_to_mount_args(compose, volume, srv_name, cnt['name'])
+        return ['--mount', args]
 
 def container_to_args(compose, cnt, detached=True, podman_command='run'):
     # TODO: double check -e , --add-host, -v, --read-only
     dirname = compose.dirname
     shared_vols = compose.shared_vols
-    pod = cnt.get('pod') or ''
+    pod = cnt.get('pod', None) or ''
     podman_args = [
         podman_command,
-        '--name={}'.format(cnt.get('name')),
+        '--name={}'.format(cnt.get('name', None)),
     ]
 
     if detached:
@@ -472,14 +514,14 @@ def container_to_args(compose, cnt, detached=True, podman_command='run'):
 
     if pod:
         podman_args.append('--pod={}'.format(pod))
-    sec = norm_as_list(cnt.get("security_opt"))
+    sec = norm_as_list(cnt.get("security_opt", None))
     for s in sec:
         podman_args.extend(['--security-opt', s])
-    if cnt.get('read_only'):
+    if cnt.get('read_only', None):
         podman_args.append('--read-only')
     for i in cnt.get('labels', []):
         podman_args.extend(['--label', i])
-    net = cnt.get("network_mode")
+    net = cnt.get("network_mode", None)
     if net:
         podman_args.extend(['--network', net])
     env = norm_as_list(cnt.get('environment', {}))
@@ -491,41 +533,40 @@ def container_to_args(compose, cnt, detached=True, podman_command='run'):
         i = os.path.realpath(os.path.join(dirname, i))
         podman_args.extend(['--env-file', i])
     tmpfs_ls = cnt.get('tmpfs', [])
-    if is_str(tmpfs_ls): tmpfs_ls=[tmpfs_ls]
+    if is_str(tmpfs_ls): tmpfs_ls = [tmpfs_ls]
     for i in tmpfs_ls:
         podman_args.extend(['--tmpfs', i])
     for volume in cnt.get('volumes', []):
         # TODO: should we make it os.path.realpath(os.path.join(, i))?
-        mount_args = mount_desc_to_args(compose, volume, cnt['_service'], cnt['name'])
+        podman_args.extend(get_mount_args(compose, cnt, volume))
-        podman_args.extend(['--mount', mount_args])
     for i in cnt.get('extra_hosts', []):
         podman_args.extend(['--add-host', i])
     for i in cnt.get('expose', []):
         podman_args.extend(['--expose', i])
-    if cnt.get('publishall'):
+    if cnt.get('publishall', None):
         podman_args.append('-P')
     for i in cnt.get('ports', []):
         podman_args.extend(['-p', i])
-    user = cnt.get('user')
+    user = cnt.get('user', None)
     if user is not None:
         podman_args.extend(['-u', user])
-    if cnt.get('working_dir') is not None:
+    if cnt.get('working_dir', None) is not None:
-        podman_args.extend(['-w', cnt.get('working_dir')])
+        podman_args.extend(['-w', cnt['working_dir']])
-    if cnt.get('hostname'):
+    if cnt.get('hostname', None):
-        podman_args.extend(['--hostname', cnt.get('hostname')])
+        podman_args.extend(['--hostname', cnt['hostname']])
-    if cnt.get('shm_size'):
+    if cnt.get('shm_size', None):
-        podman_args.extend(['--shm_size', '{}'.format(cnt.get('shm_size'))])
+        podman_args.extend(['--shm_size', '{}'.format(cnt['shm_size'])])
-    if cnt.get('stdin_open'):
+    if cnt.get('stdin_open', None):
         podman_args.append('-i')
-    if cnt.get('tty'):
+    if cnt.get('tty', None):
         podman_args.append('--tty')
-    if cnt.get('privileged'):
+    if cnt.get('privileged', None):
         podman_args.append('--privileged')
     container_to_ulimit_args(cnt, podman_args)
     # currently podman shipped by fedora does not package this
-    # if cnt.get('init'):
+    # if cnt.get('init', None):
     #    args.append('--init')
-    entrypoint = cnt.get('entrypoint')
+    entrypoint = cnt.get('entrypoint', None)
     if entrypoint is not None:
         if is_str(entrypoint):
             podman_args.extend(['--entrypoint', entrypoint])
@@ -536,7 +577,7 @@ def container_to_args(compose, cnt, detached=True, podman_command='run'):
     healthcheck = cnt.get('healthcheck', None) or {}
     if not is_dict(healthcheck):
         raise ValueError("'healthcheck' must be an key-value mapping")
-    healthcheck_test = healthcheck.get('test')
+    healthcheck_test = healthcheck.get('test', None)
     if healthcheck_test:
         # If it's a string, it's equivalent to specifying CMD-SHELL
         if is_str(healthcheck_test):
@@ -576,8 +617,8 @@ def container_to_args(compose, cnt, detached=True, podman_command='run'):
     if 'retries' in healthcheck:
         podman_args.extend(['--healthcheck-retries', '{}'.format(healthcheck['retries'])])
 
-    podman_args.append(cnt.get('image'))  # command, ..etc.
+    podman_args.append(cnt['image'])  # command, ..etc.
-    command = cnt.get('command')
+    command = cnt.get('command', None)
     if command is not None:
         if is_str(command):
             podman_args.extend(shlex.split(command))
@@ -594,9 +635,9 @@ def rec_deps(services, service_name, start_point=None):
     deps = services[service_name]["_deps"]
     for dep_name in deps.copy():
         # avoid A depens on A
-        if dep_name==service_name:
+        if dep_name == service_name:
             continue
-        dep_srv = services.get(dep_name)
+        dep_srv = services.get(dep_name, None)
         if not dep_srv:
             continue
         # NOTE: avoid creating loops, A->B->A
@@ -745,6 +786,7 @@ class PodmanCompose:
         self.shared_vols = None
         self.container_names_by_service = None
         self.container_by_name = None
+        self._prefer_volume_over_mount = True
 
     def run(self):
         args = self._parse_args()
@@ -814,7 +856,7 @@ class PodmanCompose:
                 dotenv_dict = dict([l.split("=", 1) for l in dotenv_ls if "=" in l])
         else:
             dotenv_dict = {}
-        compose={'_dirname': dirname}
+        compose = {'_dirname': dirname}
         for filename in files:
             with open(filename, 'r') as f:
                 content = yaml.safe_load(f)
@@ -826,8 +868,8 @@ class PodmanCompose:
         # debug mode
         if len(files)>1:
             print(" ** merged:\n", json.dumps(compose, indent = 2))
-        ver = compose.get('version')
+        ver = compose.get('version', None)
-        services = compose.get('services')
+        services = compose.get('services', None)
         # NOTE: maybe add "extends.service" to _deps at this stage
         flat_deps(services, with_extends=True)
         service_names = sorted([ (len(srv["_deps"]), name) for name, srv in services.items() ])
@@ -874,7 +916,7 @@ class PodmanCompose:
                         project_name=project_name,
                         service_name=service_name,
                     )
-                labels = norm_as_list(cnt.get('labels'))
+                labels = norm_as_list(cnt.get('labels', None))
                 labels.extend(podman_compose_labels)
                 labels.extend([
                     "com.docker.compose.container-number={}".format(num),
@@ -888,7 +930,7 @@ class PodmanCompose:
         container_by_name = dict([(c["name"], c) for c in given_containers])
         #print("deps:", [(c["name"], c["_deps"]) for c in given_containers])
         given_containers = list(container_by_name.values())
-        given_containers.sort(key=lambda c: len(c.get('_deps') or []))
+        given_containers.sort(key=lambda c: len(c.get('_deps', None) or []))
         #print("sorted:", [c["name"] for c in given_containers])
         tr = transformations[transform_policy]
         pods, containers = tr(
@@ -978,7 +1020,7 @@ def compose_version(compose, args):
 @cmd_run(podman_compose, 'pull', 'pull stack images')
 def compose_pull(compose, args):
     for cnt in compose.containers:
-        if cnt.get('build'): continue
+        if cnt.get('build', None): continue
         compose.podman.run(["pull", cnt["image"]], sleep=0)
 
 @cmd_run(podman_compose, 'push', 'push stack images')
@@ -1029,7 +1071,7 @@ def create_pods(compose, args):
             "--name={}".format(pod["name"]),
             "--share", "net",
         ]
-        ports = pod.get("ports") or []
+        ports = pod.get("ports", None) or []
         for i in ports:
             podman_args.extend(['-p', i])
         compose.podman.run(podman_args)
@@ -1052,8 +1094,7 @@ def compose_up(compose, args):
         # `podman build` does not cache, so don't always build
         build_args = argparse.Namespace(
             if_not_exists=(not args.build),
-            **args.__dict__,
+            **args.__dict__)
-        )
         compose.commands['build'](compose, build_args)
 
     shared_vols = compose.shared_vols