diff --git a/podman_compose.py b/podman_compose.py index 866a7e7..c023167 100755 --- a/podman_compose.py +++ b/podman_compose.py @@ -34,7 +34,7 @@ except ImportError: import json import yaml -__version__ = '0.1.6dev' +__version__ = '0.1.7dev' PY3 = sys.version_info[0] == 3 if PY3: @@ -46,7 +46,7 @@ is_str = lambda s: isinstance(s, basestring) is_dict = lambda d: isinstance(d, dict) is_list = lambda l: not is_str(l) and not is_dict(l) and hasattr(l, "__iter__") # identity filter -filteri = lambda a: filter(lambda i:i, a) +filteri = lambda a: filter(lambda i: i, a) def try_int(i, fallback=None): try: @@ -58,27 +58,24 @@ def try_int(i, fallback=None): return fallback dir_re = re.compile("^[~/\.]") -propagation_re=re.compile("^(?:z|Z|r?shared|r?slave|r?private)$") - -# NOTE: if a named volume is used but not defined it gives -# ERROR: Named volume "so and so" is used in service "xyz" but no declaration was found in the volumes section. -# unless it's anon-volume +propagation_re = re.compile("^(?:z|Z|r?shared|r?slave|r?private)$") def parse_short_mount(mount_str, basedir): mount_a = mount_str.split(':') mount_opt_dict = {} mount_opt = None - if len(mount_a)==1: - # Just specify a path and let the Engine create a volume + if len(mount_a) == 1: + # Anonymous: Just specify a path and let the engine creates the volume # - /var/lib/mysql - mount_src, mount_dst=None, mount_str - elif len(mount_a)==2: + mount_src, mount_dst = None, mount_str + elif len(mount_a) == 2: mount_src, mount_dst = mount_a - # dest must start with /, otherwise it's option + # dest must start with / like /foo:/var/lib/mysql + # otherwise it's option like /var/lib/mysql:rw if not mount_dst.startswith('/'): mount_dst, mount_opt = mount_a mount_src = None - elif len(mount_a)==3: + elif len(mount_a) == 3: mount_src, mount_dst, mount_opt = mount_a else: raise ValueError("could not parse mount "+mount_str) @@ -98,24 +95,31 @@ def parse_short_mount(mount_str, basedir): mount_type = "volume" mount_opts = filteri((mount_opt or '').split(',')) for opt in mount_opts: - if opt=='ro': mount_opt_dict["read_only"]=True - elif opt=='rw': mount_opt_dict["read_only"]=False - elif opt=='delegated': mount_opt_dict["delegated"]=dict(propagation=opt) - elif opt=='cached': mount_opt_dict["cached"]=dict(propagation=opt) - elif propagation_re.match(opt): mount_opt_dict["bind"]=dict(propagation=opt) + if opt == 'ro': mount_opt_dict["read_only"] = True + elif opt == 'rw': mount_opt_dict["read_only"] = False + elif opt in ('consistent', 'delegated', 'cached'): + mount_opt_dict["consistency"] = opt + elif propagation_re.match(opt): mount_opt_dict["bind"] = dict(propagation=opt) else: # TODO: ignore raise ValueError("unknown mount option "+opt) return dict(type=mount_type, source=mount_src, target=mount_dst, **mount_opt_dict) +# NOTE: if a named volume is used but not defined it +# gives ERROR: Named volume "abc" is used in service "xyz" +# but no declaration was found in the volumes section. +# unless it's anonymous-volume + def fix_mount_dict(mount_dict, proj_name, srv_name): """ in-place fix mount dictionary to: - add missing source - prefix source with proj_name """ - if mount_dict["type"]=="volume": - source = mount_dict.get("source") + # if already applied nothing todo + if "_source" in mount_dict: return mount_dict + if mount_dict["type"] == "volume": + source = mount_dict.get("source", None) # keep old source mount_dict["_source"] = source if not source: @@ -152,7 +156,7 @@ def dicts_get(dicts, key, fallback='', fallback_empty=False): """ value = None for d in dicts: - value = d.get(key) + value = d.get(key, None) if value is not None: break if not value: if fallback_empty or value is None: @@ -218,8 +222,8 @@ def norm_ulimit(inner_value): if is_dict(inner_value): if not inner_value.keys() & {"soft", "hard"}: raise ValueError("expected at least one soft or hard limit") - soft = inner_value.get("soft", inner_value.get("hard")) - hard = inner_value.get("hard", inner_value.get("soft")) + soft = inner_value.get("soft", inner_value.get("hard", None)) + hard = inner_value.get("hard", inner_value.get("soft", None)) return "{}:{}".format(soft, hard) elif is_list(inner_value): return norm_ulimit(norm_as_dict(inner_value)) # if int or string return as is @@ -252,9 +256,9 @@ def move_list(dst, containers, key): """ move key (like port forwarding) from containers to dst (a pod or a infra container) """ - a = set(dst.get(key) or []) + a = set(dst.get(key, None) or []) for cnt in containers: - a0 = cnt.get(key) + a0 = cnt.get(key, None) if a0: a.update(a0) del cnt[key] @@ -326,7 +330,7 @@ def tr_cntnet(project_name, services, given_containers): ) for cnt0 in given_containers: cnt = dict(cnt0, network_mode="container:"+infra_name) - deps = cnt.get("depends_on") or [] + deps = cnt.get("depends_on", None) or [] deps.append(infra_name) cnt["depends_on"] = deps # adjust hosts to point to localhost, TODO: adjust host env @@ -366,73 +370,60 @@ def tr_1podfw(project_name, services, given_containers): return pods, containers -def mount_dict_vol_to_bind(compose, mount_dict): +def assert_volume(compose, mount_dict): """ inspect volume to get directory create volume if needed - and return mount_dict as bind of that directory """ + if mount_dict["type"] != "volume": return proj_name = compose.project_name shared_vols = compose.shared_vols - if mount_dict["type"]!="volume": return mount_dict + vol_name_orig = mount_dict.get("_source", None) vol_name = mount_dict["source"] print("podman volume inspect {vol_name} || podman volume create {vol_name}".format(vol_name=vol_name)) + # TODO: might move to using "volume list" # podman volume list --format '{{.Name}}\t{{.MountPoint}}' -f 'label=io.podman.compose.project=HERE' try: out = compose.podman.output(["volume", "inspect", vol_name]).decode('utf-8') except subprocess.CalledProcessError: compose.podman.output(["volume", "create", "--label", "io.podman.compose.project={}".format(proj_name), vol_name]) out = compose.podman.output(["volume", "inspect", vol_name]).decode('utf-8') - try: - src = json.loads(out)[0]["mountPoint"] - except KeyError: - src = json.loads(out)[0]["Mountpoint"] - ret=dict(mount_dict, type="bind", source=src, _vol=vol_name) - bind_prop=ret.get("bind", {}).get("propagation") - if not bind_prop: - if "bind" not in ret: - ret["bind"]={} - # if in top level volumes then it's shared bind-propagation=z - if vol_name_orig and vol_name_orig in shared_vols: - ret["bind"]["propagation"]="z" - else: - ret["bind"]["propagation"]="Z" - try: del ret["volume"] - except KeyError: pass - return ret -def mount_desc_to_args(compose, mount_desc, srv_name, cnt_name): +def mount_desc_to_mount_args(compose, mount_desc, srv_name, cnt_name): basedir = compose.dirname proj_name = compose.project_name shared_vols = compose.shared_vols - if is_str(mount_desc): mount_desc=parse_short_mount(mount_desc, basedir) - # not needed - # podman support: podman run --rm -ti --mount type=volume,source=myvol,destination=/delme busybox - mount_desc = mount_dict_vol_to_bind(compose, fix_mount_dict(mount_desc, proj_name, srv_name)) - mount_type = mount_desc.get("type") - source = mount_desc.get("source") + mount_type = mount_desc.get("type", None) + source = mount_desc.get("source", None) target = mount_desc["target"] - opts=[] - if mount_desc.get("bind"): - bind_prop=mount_desc["bind"].get("propagation") - if bind_prop: opts.append("bind-propagation={}".format(bind_prop)) + opts = [] + if mount_desc.get(mount_type, None): + # TODO: we might need to add mount_dict[mount_type]["propagation"] = "z" + mount_prop = mount_desc.get(mount_type, {}).get("propagation", None) + if mount_prop: opts.append("{}-propagation={}".format(mount_type, mount_prop)) if mount_desc.get("read_only", False): opts.append("ro") - if mount_type=='tmpfs': + if mount_type == 'tmpfs': tmpfs_opts = mount_desc.get("tmpfs", {}) - tmpfs_size = tmpfs_opts.get("size") + tmpfs_size = tmpfs_opts.get("size", None) if tmpfs_size: opts.append("tmpfs-size={}".format(tmpfs_size)) - tmpfs_mode = tmpfs_opts.get("mode") + tmpfs_mode = tmpfs_opts.get("mode", None) if tmpfs_mode: opts.append("tmpfs-mode={}".format(tmpfs_mode)) - opts=",".join(opts) - if mount_type=='bind': + opts = ",".join(opts) + if mount_type == 'bind': return "type=bind,source={source},destination={target},{opts}".format( source=source, target=target, opts=opts ).rstrip(",") - elif mount_type=='tmpfs': + elif mount_type == 'volume': + return "type=volume,source={source},destination={target},{opts}".format( + source=source, + target=target, + opts=opts + ).rstrip(",") + elif mount_type == 'tmpfs': return "type=tmpfs,destination={target},{opts}".format( target=target, opts=opts @@ -440,8 +431,6 @@ def mount_desc_to_args(compose, mount_desc, srv_name, cnt_name): else: raise ValueError("unknown mount type:"+mount_type) - - def container_to_ulimit_args(cnt, podman_args): ulimit = cnt.get('ulimits', []) if ulimit is not None: @@ -455,16 +444,69 @@ def container_to_ulimit_args(cnt, podman_args): for i in ulimit: podman_args.extend(['--ulimit', i]) +def mount_desc_to_volume_args(compose, mount_desc, srv_name, cnt_name): + basedir = compose.dirname + proj_name = compose.project_name + shared_vols = compose.shared_vols + mount_type = mount_desc["type"] + source = mount_desc.get("source", None) + target = mount_desc["target"] + opts = [] + if mount_type != 'bind' and mount_type != 'volume': + raise ValueError("unknown mount type:"+mount_type) + propagations = set(filteri(mount_desc.get(mount_type, {}).get("propagation", "").split(','))) + if mount_type != 'bind': + propagations.update(filteri(mount_desc.get('bind', {}).get("propagation", "").split(','))) + opts.extend(propagations) + # --volume, -v[=[[SOURCE-VOLUME|HOST-DIR:]CONTAINER-DIR[:OPTIONS]]] + # [rw|ro] + # [z|Z] + # [[r]shared|[r]slave|[r]private] + # [[r]bind] + # [noexec|exec] + # [nodev|dev] + # [nosuid|suid] + read_only = mount_desc.get("read_only", None) + if read_only is not None: + opts.append('ro' if read_only else 'rw') + args = f'{source}:{target}' + if opts: args += ':' + ','.join(opts) + return args +def get_mount_args(compose, cnt, volume): + proj_name = compose.project_name + srv_name = cnt['_service'] + basedir = compose.dirname + if is_str(volume): volume = parse_short_mount(volume, basedir) + mount_type = volume["type"] + assert_volume(compose, fix_mount_dict(volume, proj_name, srv_name)) + if compose._prefer_volume_over_mount: + if mount_type == 'tmpfs': + # TODO: --tmpfs /tmp:rw,size=787448k,mode=1777 + args = volume['target'] + tmpfs_opts = volume.get("tmpfs", {}) + opts = [] + size = tmpfs_opts.get("size", None) + if size: opts.append('size={}'.format(size)) + mode = tmpfs_opts.get("mode", None) + if mode: opts.append('mode={}'.format(size)) + if opts: args += ':' + ','.join(opts) + return ['--tmpfs', args] + else: + args = mount_desc_to_volume_args(compose, volume, srv_name, cnt['name']) + return ['-v', args] + else: + args = mount_desc_to_mount_args(compose, volume, srv_name, cnt['name']) + return ['--mount', args] def container_to_args(compose, cnt, detached=True, podman_command='run'): # TODO: double check -e , --add-host, -v, --read-only dirname = compose.dirname shared_vols = compose.shared_vols - pod = cnt.get('pod') or '' + pod = cnt.get('pod', None) or '' podman_args = [ podman_command, - '--name={}'.format(cnt.get('name')), + '--name={}'.format(cnt.get('name', None)), ] if detached: @@ -472,14 +514,14 @@ def container_to_args(compose, cnt, detached=True, podman_command='run'): if pod: podman_args.append('--pod={}'.format(pod)) - sec = norm_as_list(cnt.get("security_opt")) + sec = norm_as_list(cnt.get("security_opt", None)) for s in sec: podman_args.extend(['--security-opt', s]) - if cnt.get('read_only'): + if cnt.get('read_only', None): podman_args.append('--read-only') for i in cnt.get('labels', []): podman_args.extend(['--label', i]) - net = cnt.get("network_mode") + net = cnt.get("network_mode", None) if net: podman_args.extend(['--network', net]) env = norm_as_list(cnt.get('environment', {})) @@ -491,41 +533,40 @@ def container_to_args(compose, cnt, detached=True, podman_command='run'): i = os.path.realpath(os.path.join(dirname, i)) podman_args.extend(['--env-file', i]) tmpfs_ls = cnt.get('tmpfs', []) - if is_str(tmpfs_ls): tmpfs_ls=[tmpfs_ls] + if is_str(tmpfs_ls): tmpfs_ls = [tmpfs_ls] for i in tmpfs_ls: podman_args.extend(['--tmpfs', i]) for volume in cnt.get('volumes', []): # TODO: should we make it os.path.realpath(os.path.join(, i))? - mount_args = mount_desc_to_args(compose, volume, cnt['_service'], cnt['name']) - podman_args.extend(['--mount', mount_args]) + podman_args.extend(get_mount_args(compose, cnt, volume)) for i in cnt.get('extra_hosts', []): podman_args.extend(['--add-host', i]) for i in cnt.get('expose', []): podman_args.extend(['--expose', i]) - if cnt.get('publishall'): + if cnt.get('publishall', None): podman_args.append('-P') for i in cnt.get('ports', []): podman_args.extend(['-p', i]) - user = cnt.get('user') + user = cnt.get('user', None) if user is not None: podman_args.extend(['-u', user]) - if cnt.get('working_dir') is not None: - podman_args.extend(['-w', cnt.get('working_dir')]) - if cnt.get('hostname'): - podman_args.extend(['--hostname', cnt.get('hostname')]) - if cnt.get('shm_size'): - podman_args.extend(['--shm_size', '{}'.format(cnt.get('shm_size'))]) - if cnt.get('stdin_open'): + if cnt.get('working_dir', None) is not None: + podman_args.extend(['-w', cnt['working_dir']]) + if cnt.get('hostname', None): + podman_args.extend(['--hostname', cnt['hostname']]) + if cnt.get('shm_size', None): + podman_args.extend(['--shm_size', '{}'.format(cnt['shm_size'])]) + if cnt.get('stdin_open', None): podman_args.append('-i') - if cnt.get('tty'): + if cnt.get('tty', None): podman_args.append('--tty') - if cnt.get('privileged'): + if cnt.get('privileged', None): podman_args.append('--privileged') container_to_ulimit_args(cnt, podman_args) # currently podman shipped by fedora does not package this - # if cnt.get('init'): + # if cnt.get('init', None): # args.append('--init') - entrypoint = cnt.get('entrypoint') + entrypoint = cnt.get('entrypoint', None) if entrypoint is not None: if is_str(entrypoint): podman_args.extend(['--entrypoint', entrypoint]) @@ -536,7 +577,7 @@ def container_to_args(compose, cnt, detached=True, podman_command='run'): healthcheck = cnt.get('healthcheck', None) or {} if not is_dict(healthcheck): raise ValueError("'healthcheck' must be an key-value mapping") - healthcheck_test = healthcheck.get('test') + healthcheck_test = healthcheck.get('test', None) if healthcheck_test: # If it's a string, it's equivalent to specifying CMD-SHELL if is_str(healthcheck_test): @@ -576,8 +617,8 @@ def container_to_args(compose, cnt, detached=True, podman_command='run'): if 'retries' in healthcheck: podman_args.extend(['--healthcheck-retries', '{}'.format(healthcheck['retries'])]) - podman_args.append(cnt.get('image')) # command, ..etc. - command = cnt.get('command') + podman_args.append(cnt['image']) # command, ..etc. + command = cnt.get('command', None) if command is not None: if is_str(command): podman_args.extend(shlex.split(command)) @@ -594,9 +635,9 @@ def rec_deps(services, service_name, start_point=None): deps = services[service_name]["_deps"] for dep_name in deps.copy(): # avoid A depens on A - if dep_name==service_name: + if dep_name == service_name: continue - dep_srv = services.get(dep_name) + dep_srv = services.get(dep_name, None) if not dep_srv: continue # NOTE: avoid creating loops, A->B->A @@ -745,6 +786,7 @@ class PodmanCompose: self.shared_vols = None self.container_names_by_service = None self.container_by_name = None + self._prefer_volume_over_mount = True def run(self): args = self._parse_args() @@ -814,7 +856,7 @@ class PodmanCompose: dotenv_dict = dict([l.split("=", 1) for l in dotenv_ls if "=" in l]) else: dotenv_dict = {} - compose={'_dirname': dirname} + compose = {'_dirname': dirname} for filename in files: with open(filename, 'r') as f: content = yaml.safe_load(f) @@ -826,8 +868,8 @@ class PodmanCompose: # debug mode if len(files)>1: print(" ** merged:\n", json.dumps(compose, indent = 2)) - ver = compose.get('version') - services = compose.get('services') + ver = compose.get('version', None) + services = compose.get('services', None) # NOTE: maybe add "extends.service" to _deps at this stage flat_deps(services, with_extends=True) service_names = sorted([ (len(srv["_deps"]), name) for name, srv in services.items() ]) @@ -874,7 +916,7 @@ class PodmanCompose: project_name=project_name, service_name=service_name, ) - labels = norm_as_list(cnt.get('labels')) + labels = norm_as_list(cnt.get('labels', None)) labels.extend(podman_compose_labels) labels.extend([ "com.docker.compose.container-number={}".format(num), @@ -888,7 +930,7 @@ class PodmanCompose: container_by_name = dict([(c["name"], c) for c in given_containers]) #print("deps:", [(c["name"], c["_deps"]) for c in given_containers]) given_containers = list(container_by_name.values()) - given_containers.sort(key=lambda c: len(c.get('_deps') or [])) + given_containers.sort(key=lambda c: len(c.get('_deps', None) or [])) #print("sorted:", [c["name"] for c in given_containers]) tr = transformations[transform_policy] pods, containers = tr( @@ -978,7 +1020,7 @@ def compose_version(compose, args): @cmd_run(podman_compose, 'pull', 'pull stack images') def compose_pull(compose, args): for cnt in compose.containers: - if cnt.get('build'): continue + if cnt.get('build', None): continue compose.podman.run(["pull", cnt["image"]], sleep=0) @cmd_run(podman_compose, 'push', 'push stack images') @@ -1029,7 +1071,7 @@ def create_pods(compose, args): "--name={}".format(pod["name"]), "--share", "net", ] - ports = pod.get("ports") or [] + ports = pod.get("ports", None) or [] for i in ports: podman_args.extend(['-p', i]) compose.podman.run(podman_args) @@ -1052,8 +1094,7 @@ def compose_up(compose, args): # `podman build` does not cache, so don't always build build_args = argparse.Namespace( if_not_exists=(not args.build), - **args.__dict__, - ) + **args.__dict__) compose.commands['build'](compose, build_args) shared_vols = compose.shared_vols