From f0f51057d6347f3924b9abf66a4f3124349414e7 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 13:30:30 -0500 Subject: [PATCH 01/23] hint how to list valid env configName(s) --- CHANGELOG.md | 4 ++++ cmd/zrok/configGet.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2933da2d..afa20c85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## v0.4.47 + +CHANGE: Add usage hint in `zrok config get --help` to clarify how to list all valid `configName` and their current values by running `zrok status`. + ## v0.4.46 FEATURE: Linux service template for systemd user units (https://github.com/openziti/zrok/pull/818) diff --git a/cmd/zrok/configGet.go b/cmd/zrok/configGet.go index 0d1905d0..53145687 100644 --- a/cmd/zrok/configGet.go +++ b/cmd/zrok/configGet.go @@ -17,7 +17,7 @@ type configGetCommand struct { func newConfigGetCommand() *configGetCommand { cmd := &cobra.Command{ Use: "get ", - Short: "Get a value from the environment config", + Short: "Get a value from the environment config. Run 'zrok status' to list valid config names and their current values.", Args: cobra.ExactArgs(1), } command := &configGetCommand{cmd: cmd} From 2f009957fc9324d45a33e80be6b6f0b419d202c7 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 14:53:47 -0500 Subject: [PATCH 02/23] add troubleshooting help for dns propagation error --- docker/compose/zrok-instance/README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/compose/zrok-instance/README.md b/docker/compose/zrok-instance/README.md index ba68d0d7..73992947 100644 --- a/docker/compose/zrok-instance/README.md +++ b/docker/compose/zrok-instance/README.md @@ -190,7 +190,7 @@ See "My internet connection can only send traffic to common ports" below about c docker compose logs zrok-controller ``` -1. Check the caddy logs. +1. Check the Caddy logs. It can take a few minutes for Caddy to obtain the wildcard certificate. You can check the logs to see if there were any errors completing the DNS challenge which involves using the Caddy DNS plugin to create a TXT record in your DNS zone. This leverages the API token you provided in the `.env` file, which must have permission to create DNS records in the zrok DNS zone. @@ -198,6 +198,23 @@ See "My internet connection can only send traffic to common ports" below about c docker compose logs caddy ``` +1. Caddy keeps failing to obtain a wildcard certificate because it timed out waiting for DNS. + + Symptom: the Caddy log contains "timed out waiting for record to fully propagate." This means that Caddy added a DNS record with your DNS provider's API to prove to the CA it controls the zrok DNS zone, but it wasn't able to verify the record was created successfully with a DNS query. + + Solutions: + + - Add `propagation_delay` in your `Caddyfile` to delay the first DNS verification query. This avoids caching a verification query failure by waiting a few minutes for the record to become available so the verification query will succeed on the first attempt. Caddy will be unable to verify the DNS record if the failure remains in the cache too long. + - If the prior solution fails, you can override the default resolves/nameservers with `resolvers`, a space-separated list of DNS servers. This gives you more control over if and where the verification query result is cached. + + ``` + tls { + dns {CADDY_DNS_PLUGIN} {CADDY_DNS_PLUGIN_TOKEN} + propagation_timeout 60m # default 2m + propagation_delay 5m # default 0m + } + ``` + 1. `zrok enable` fails certificate verification: ensure you are not using the staging API for Let's Encrypt. If you are using the staging API, you will see an error about the API certificate when you use the zrok CLI. You can switch to the production API by removing the overriding assignment of the `CADDY_ACME_API` variable. From 6dd441a6d60f8fc0e2c687831320a9abefa6a2db Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 14:55:46 -0500 Subject: [PATCH 03/23] announce Docker instance doc update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2933da2d..77cfb6fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## v0.4.47 + +CHANGE: Document solving the DNS propagation timeout for Docker instances that are using Caddy to manage the wildcard certificate. + ## v0.4.46 FEATURE: Linux service template for systemd user units (https://github.com/openziti/zrok/pull/818) From 9b067d22668868c67f3ddd3396da31b8ff1ad311 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 15:05:35 -0500 Subject: [PATCH 04/23] use cobra long messages instead of crowding the short messages with too much info --- cmd/zrok/configGet.go | 3 ++- cmd/zrok/configSet.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/zrok/configGet.go b/cmd/zrok/configGet.go index 53145687..e095ca99 100644 --- a/cmd/zrok/configGet.go +++ b/cmd/zrok/configGet.go @@ -17,7 +17,8 @@ type configGetCommand struct { func newConfigGetCommand() *configGetCommand { cmd := &cobra.Command{ Use: "get ", - Short: "Get a value from the environment config. Run 'zrok status' to list valid config names and their current values.", + Short: "Get a value from the environment config", + Long: "Get a value from the environment config. Use 'zrok status' to list available configuration names and current values", Args: cobra.ExactArgs(1), } command := &configGetCommand{cmd: cmd} diff --git a/cmd/zrok/configSet.go b/cmd/zrok/configSet.go index 44773fbf..49846c1c 100644 --- a/cmd/zrok/configSet.go +++ b/cmd/zrok/configSet.go @@ -22,6 +22,7 @@ func newConfigSetCommand() *configSetCommand { cmd := &cobra.Command{ Use: "set ", Short: "Set a value into the environment config", + Long: "Set a value into the environment config. Use 'zrok status' to list available configuration names and current values", Args: cobra.ExactArgs(2), } command := &configSetCommand{cmd: cmd} From a1f72c061b97a78b5d7f874abd297bb713459e52 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 15:08:08 -0500 Subject: [PATCH 05/23] punctuate consistently --- cmd/zrok/configGet.go | 2 +- cmd/zrok/configSet.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/zrok/configGet.go b/cmd/zrok/configGet.go index e095ca99..6706d206 100644 --- a/cmd/zrok/configGet.go +++ b/cmd/zrok/configGet.go @@ -18,7 +18,7 @@ func newConfigGetCommand() *configGetCommand { cmd := &cobra.Command{ Use: "get ", Short: "Get a value from the environment config", - Long: "Get a value from the environment config. Use 'zrok status' to list available configuration names and current values", + Long: "Get a value from the environment config. Use 'zrok status' to list available configuration names and current values.", Args: cobra.ExactArgs(1), } command := &configGetCommand{cmd: cmd} diff --git a/cmd/zrok/configSet.go b/cmd/zrok/configSet.go index 49846c1c..76c9f3d7 100644 --- a/cmd/zrok/configSet.go +++ b/cmd/zrok/configSet.go @@ -22,7 +22,7 @@ func newConfigSetCommand() *configSetCommand { cmd := &cobra.Command{ Use: "set ", Short: "Set a value into the environment config", - Long: "Set a value into the environment config. Use 'zrok status' to list available configuration names and current values", + Long: "Set a value into the environment config. Use 'zrok status' to list available configuration names and current values.", Args: cobra.ExactArgs(2), } command := &configSetCommand{cmd: cmd} From 9ee3db852a4455227d6abfecf963cace1ad572de Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 15:19:27 -0500 Subject: [PATCH 06/23] tidy py sdk examples --- sdk/python/examples/http-server/README.md | 2 +- sdk/python/examples/pastebin/pastebin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/examples/http-server/README.md b/sdk/python/examples/http-server/README.md index a8d9088d..84912a32 100644 --- a/sdk/python/examples/http-server/README.md +++ b/sdk/python/examples/http-server/README.md @@ -4,7 +4,7 @@ This `http-server` example is a minimal zrok application that surfaces a basic H ## Implementation -```go +```python root = zrok.environment.root.Load() ``` diff --git a/sdk/python/examples/pastebin/pastebin.py b/sdk/python/examples/pastebin/pastebin.py index 96cc2391..4de0dd57 100755 --- a/sdk/python/examples/pastebin/pastebin.py +++ b/sdk/python/examples/pastebin/pastebin.py @@ -101,4 +101,4 @@ if __name__ == "__main__": server_thread = threading.Thread(target=options.func, args=[options]) server_thread.start() - server_thread.join() \ No newline at end of file + server_thread.join() From 701f89767882efe8dce9114b5333debe1b83e84f Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 15:20:21 -0500 Subject: [PATCH 07/23] make overview a class with data types instead of a function that returns JSON --- sdk/python/sdk/zrok/zrok/overview.py | 87 +++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 16 deletions(-) diff --git a/sdk/python/sdk/zrok/zrok/overview.py b/sdk/python/sdk/zrok/zrok/overview.py index b8d64649..0ee063dd 100644 --- a/sdk/python/sdk/zrok/zrok/overview.py +++ b/sdk/python/sdk/zrok/zrok/overview.py @@ -1,20 +1,75 @@ -from zrok.environment.root import Root +import json +from dataclasses import dataclass, field +from typing import List + import urllib3 +from zrok.environment.root import Root + +from zrok_api.models.environment import Environment +from zrok_api.models.share import Share -def Overview(root: Root) -> str: - if not root.IsEnabled(): - raise Exception("environment is not enabled; enable with 'zrok enable' first!") +@dataclass +class EnvironmentAndShares: + environment: Environment + shares: List[Share] = field(default_factory=list) - http = urllib3.PoolManager() - apiEndpoint = root.ApiEndpoint().endpoint - try: - response = http.request( - 'GET', - apiEndpoint + "/api/v1/overview", - headers={ - "X-TOKEN": root.env.Token - }) - except Exception as e: - raise Exception("unable to get account overview", e) - return response.data.decode('utf-8') + +@dataclass +class Overview: + environments: List[EnvironmentAndShares] = field(default_factory=list) + + @classmethod + def create(cls, root: Root) -> 'Overview': + if not root.IsEnabled(): + raise Exception("environment is not enabled; enable with 'zrok enable' first!") + + http = urllib3.PoolManager() + apiEndpoint = root.ApiEndpoint().endpoint + try: + response = http.request( + 'GET', + apiEndpoint + "/api/v1/overview", + headers={ + "X-TOKEN": root.env.Token + }) + except Exception as e: + raise Exception("unable to get account overview", e) + + json_data = json.loads(response.data.decode('utf-8')) + overview = cls() + + for env_data in json_data.get('environments', []): + env_dict = env_data['environment'] + # Map the JSON keys to the Environment class parameters + environment = Environment( + description=env_dict.get('description'), + host=env_dict.get('host'), + address=env_dict.get('address'), + z_id=env_dict.get('zId'), + activity=env_dict.get('activity'), + limited=env_dict.get('limited'), + created_at=env_dict.get('createdAt'), + updated_at=env_dict.get('updatedAt') + ) + # Map the JSON keys to the Share class parameters + shares = [] + for share_data in env_data.get('shares', []): + share = Share( + token=share_data.get('token'), + z_id=share_data.get('zId'), + share_mode=share_data.get('shareMode'), + backend_mode=share_data.get('backendMode'), + frontend_selection=share_data.get('frontendSelection'), + frontend_endpoint=share_data.get('frontendEndpoint'), + backend_proxy_endpoint=share_data.get('backendProxyEndpoint'), + reserved=share_data.get('reserved'), + activity=share_data.get('activity'), + limited=share_data.get('limited'), + created_at=share_data.get('createdAt'), + updated_at=share_data.get('updatedAt') + ) + shares.append(share) + overview.environments.append(EnvironmentAndShares(environment=environment, shares=shares)) + + return overview From e16ae7f43ee505d06cec7f33f9e8453414cc654e Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 17:57:36 -0500 Subject: [PATCH 08/23] tidy py sdk examples --- .flake8 | 5 ++++- sdk/python/examples/http-server/README.md | 2 +- sdk/python/examples/http-server/server.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.flake8 b/.flake8 index 55118b3f..eb5f618d 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,6 @@ [flake8] max-line-length = 120 -exclude = zrok_api, build \ No newline at end of file +exclude = + ./sdk/python/sdk/zrok/zrok_api/**, + ./build/** + diff --git a/sdk/python/examples/http-server/README.md b/sdk/python/examples/http-server/README.md index 84912a32..41218187 100644 --- a/sdk/python/examples/http-server/README.md +++ b/sdk/python/examples/http-server/README.md @@ -46,6 +46,6 @@ Next, we run the server which ends up calling the following: @zrok.decor.zrok(opts=zrok_opts) def runApp(): from waitress import serve - # the port is only used to integrate Zrok with frameworks that expect a "hostname:port" combo + # the port is only used to integrate zrok with frameworks that expect a "hostname:port" combo serve(app, port=bindPort) ``` diff --git a/sdk/python/examples/http-server/server.py b/sdk/python/examples/http-server/server.py index bebbba78..912584e3 100755 --- a/sdk/python/examples/http-server/server.py +++ b/sdk/python/examples/http-server/server.py @@ -13,7 +13,7 @@ bindPort = 18081 @zrok.decor.zrok(opts=zrok_opts) def runApp(): from waitress import serve - # the port is only used to integrate Zrok with frameworks that expect a "hostname:port" combo + # the port is only used to integrate zrok with frameworks that expect a "hostname:port" combo serve(app, port=bindPort) From 32611af5f40271c02cd987ff40f085ea8ce083f9 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 17:58:16 -0500 Subject: [PATCH 09/23] refactor overview class implementation to leverage existing EnvironmentAndResources model --- sdk/python/sdk/zrok/zrok/overview.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/sdk/python/sdk/zrok/zrok/overview.py b/sdk/python/sdk/zrok/zrok/overview.py index 0ee063dd..9cadae66 100644 --- a/sdk/python/sdk/zrok/zrok/overview.py +++ b/sdk/python/sdk/zrok/zrok/overview.py @@ -4,20 +4,16 @@ from typing import List import urllib3 from zrok.environment.root import Root - from zrok_api.models.environment import Environment +from zrok_api.models.environment_and_resources import EnvironmentAndResources +from zrok_api.models.frontends import Frontends from zrok_api.models.share import Share - - -@dataclass -class EnvironmentAndShares: - environment: Environment - shares: List[Share] = field(default_factory=list) +from zrok_api.models.shares import Shares @dataclass class Overview: - environments: List[EnvironmentAndShares] = field(default_factory=list) + environments: List[EnvironmentAndResources] = field(default_factory=list) @classmethod def create(cls, root: Root) -> 'Overview': @@ -52,8 +48,9 @@ class Overview: created_at=env_dict.get('createdAt'), updated_at=env_dict.get('updatedAt') ) - # Map the JSON keys to the Share class parameters - shares = [] + + # Create Shares object from share data + share_list = [] for share_data in env_data.get('shares', []): share = Share( token=share_data.get('token'), @@ -69,7 +66,14 @@ class Overview: created_at=share_data.get('createdAt'), updated_at=share_data.get('updatedAt') ) - shares.append(share) - overview.environments.append(EnvironmentAndShares(environment=environment, shares=shares)) + share_list.append(share) + + # Create EnvironmentAndResources object + env_resources = EnvironmentAndResources( + environment=environment, + shares=share_list, + frontends=Frontends() # Empty frontends for now as it's not in the input data + ) + overview.environments.append(env_resources) return overview From 141d219ef7e87fa235ae89d04b169cce41f0d26a Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 17:58:37 -0500 Subject: [PATCH 10/23] add a py sdk function to release a reserved share --- sdk/python/sdk/zrok/zrok/share.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sdk/python/sdk/zrok/zrok/share.py b/sdk/python/sdk/zrok/zrok/share.py index b61af2fb..918f5c6a 100644 --- a/sdk/python/sdk/zrok/zrok/share.py +++ b/sdk/python/sdk/zrok/zrok/share.py @@ -104,3 +104,19 @@ def DeleteShare(root: Root, shr: model.Share): ShareApi(zrok).unshare(body=req) except Exception as e: raise Exception("error deleting share", e) + + +def ReleaseReservedShare(root: Root, shr: model.Share): + req = UnshareRequest(env_zid=root.env.ZitiIdentity, + shr_token=shr.Token, + reserved=True) + + try: + zrok = root.Client() + except Exception as e: + raise Exception("error getting zrok client", e) + + try: + ShareApi(zrok).unshare(body=req) + except Exception as e: + raise Exception("error releasing share", e) From 26a3d802e735f989007653f7de9ba62131e49a39 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 18:00:25 -0500 Subject: [PATCH 11/23] add a py sdk example for a proxy backend --- sdk/python/examples/proxy/proxy.py | 153 +++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 sdk/python/examples/proxy/proxy.py diff --git a/sdk/python/examples/proxy/proxy.py b/sdk/python/examples/proxy/proxy.py new file mode 100644 index 00000000..97f7d2a4 --- /dev/null +++ b/sdk/python/examples/proxy/proxy.py @@ -0,0 +1,153 @@ +import argparse +import atexit +import logging +import sys +import urllib.parse + +import requests +from flask import Flask, Response, request +from waitress import serve +from zrok.model import ShareRequest, Share +from zrok.overview import EnvironmentAndResources, Overview + +import zrok + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +app = Flask(__name__) +target_url = None +zrok_opts = {} +bindPort = 18081 + +# List of hop-by-hop headers that should not be returned to the viewer +HOP_BY_HOP_HEADERS = { + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailers', + 'transfer-encoding', + 'upgrade' +} + + +@app.route('/', defaults={'path': ''}, methods=['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS', 'PATCH']) +@app.route('/', methods=['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS', 'PATCH']) +def proxy(path): + global target_url + logger.info(f"Incoming {request.method} request to {request.path}") + logger.info(f"Headers: {dict(request.headers)}") + + # Forward the request to target URL + full_url = urllib.parse.urljoin(target_url, request.path) + logger.info(f"Forwarding to: {full_url}") + + # Copy request headers, excluding hop-by-hop headers + headers = {k: v for k, v in request.headers.items() if k.lower() not in HOP_BY_HOP_HEADERS and k.lower() != 'host'} + + try: + response = requests.request( + method=request.method, + url=full_url, + headers=headers, + data=request.get_data(), + stream=True + ) + + logger.info(f"Response status: {response.status_code}") + logger.info(f"Response headers: {dict(response.headers)}") + + # Filter out hop-by-hop headers from the response + filtered_headers = {k: v for k, v in response.headers.items() if k.lower() not in HOP_BY_HOP_HEADERS} + + return Response( + response.iter_content(chunk_size=8192), + status=response.status_code, + headers=filtered_headers + ) + + except Exception as e: + logger.error(f"Proxy error: {str(e)}", exc_info=True) + return str(e), 502 + + +@zrok.decor.zrok(opts=zrok_opts) +def run_proxy(): + # the port is only used to integrate zrok with frameworks that expect a "hostname:port" combo + serve(app, port=bindPort) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Start a zrok proxy server') + parser.add_argument('target_url', help='Target URL to proxy requests to') + parser.add_argument('-n', '--unique-name', help='Unique name for the proxy instance') + args = parser.parse_args() + + target_url = args.target_url + logger.info("=== Starting proxy server ===") + logger.info(f"Target URL: {target_url}") + logger.info(f"Logging level: {logger.getEffectiveLevel()}") + + root = zrok.environment.root.Load() + my_env = EnvironmentAndResources( + environment=None, + shares=[] + ) + overview = Overview.create(root=root) + for env_stuff in overview.environments: + if env_stuff.environment.z_id == root.env.ZitiIdentity: + my_env = EnvironmentAndResources( + environment=env_stuff.environment, + shares=env_stuff.shares + ) + break + + if my_env: + logger.debug( + f"Found environment in overview with Ziti identity " + f"matching local environment: {my_env.environment.z_id}" + ) + else: + logger.error("No matching environment found") + sys.exit(1) + + existing_reserved_share = None + for share in my_env.shares: + if share.token == args.unique_name: + existing_reserved_share = share + break + + if existing_reserved_share: + logger.debug(f"Found existing share with token: {existing_reserved_share.token}") + shr = Share(Token=existing_reserved_share.token, FrontendEndpoints=[existing_reserved_share.frontend_endpoint]) + else: + logger.debug(f"No existing share found with token: {args.unique_name}") + share_request = ShareRequest( + BackendMode=zrok.model.PROXY_BACKEND_MODE, + ShareMode=zrok.model.PUBLIC_SHARE_MODE, + Frontends=['public'], + Target="http-proxy", + Reserved=True + ) + if args.unique_name: + share_request.UniqueName = args.unique_name + + shr = zrok.share.CreateShare(root=root, request=share_request) + + def cleanup(): + zrok.share.ReleaseReservedShare(root=root, shr=shr) + logger.info(f"Share {shr.Token} released") + if not args.unique_name: + atexit.register(cleanup) + + zrok_opts['cfg'] = zrok.decor.Opts(root=root, shrToken=shr.Token, bindPort=bindPort) + + logger.info(f"Access proxy at: {', '.join(shr.FrontendEndpoints)}") + + run_proxy() From f27ab0a888f559cd70217f76258e0e56691e4dd5 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 18:08:14 -0500 Subject: [PATCH 12/23] tidy py sdk --- sdk/python/sdk/zrok/zrok/overview.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/python/sdk/zrok/zrok/overview.py b/sdk/python/sdk/zrok/zrok/overview.py index 9cadae66..f7b981eb 100644 --- a/sdk/python/sdk/zrok/zrok/overview.py +++ b/sdk/python/sdk/zrok/zrok/overview.py @@ -8,7 +8,6 @@ from zrok_api.models.environment import Environment from zrok_api.models.environment_and_resources import EnvironmentAndResources from zrok_api.models.frontends import Frontends from zrok_api.models.share import Share -from zrok_api.models.shares import Shares @dataclass @@ -48,7 +47,7 @@ class Overview: created_at=env_dict.get('createdAt'), updated_at=env_dict.get('updatedAt') ) - + # Create Shares object from share data share_list = [] for share_data in env_data.get('shares', []): From 9097643617536d091ef17e0e074bc80a41381b08 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 18:08:14 -0500 Subject: [PATCH 13/23] tidy py sdk --- sdk/python/sdk/zrok/zrok/overview.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/python/sdk/zrok/zrok/overview.py b/sdk/python/sdk/zrok/zrok/overview.py index 9cadae66..f7b981eb 100644 --- a/sdk/python/sdk/zrok/zrok/overview.py +++ b/sdk/python/sdk/zrok/zrok/overview.py @@ -8,7 +8,6 @@ from zrok_api.models.environment import Environment from zrok_api.models.environment_and_resources import EnvironmentAndResources from zrok_api.models.frontends import Frontends from zrok_api.models.share import Share -from zrok_api.models.shares import Shares @dataclass @@ -48,7 +47,7 @@ class Overview: created_at=env_dict.get('createdAt'), updated_at=env_dict.get('updatedAt') ) - + # Create Shares object from share data share_list = [] for share_data in env_data.get('shares', []): From fe056a2ccce5cdd7dd869dd5bbe8893952c4fe94 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 18:33:31 -0500 Subject: [PATCH 14/23] migrate the proxy example to ProxyShare class so it's a reusable feature of the Py SDK --- sdk/python/examples/proxy/proxy.py | 159 +++++--------------------- sdk/python/sdk/zrok/zrok/README.md | 47 ++++++++ sdk/python/sdk/zrok/zrok/proxy.py | 178 +++++++++++++++++++++++++++++ 3 files changed, 253 insertions(+), 131 deletions(-) create mode 100644 sdk/python/sdk/zrok/zrok/README.md create mode 100644 sdk/python/sdk/zrok/zrok/proxy.py diff --git a/sdk/python/examples/proxy/proxy.py b/sdk/python/examples/proxy/proxy.py index 97f7d2a4..94f3f010 100644 --- a/sdk/python/examples/proxy/proxy.py +++ b/sdk/python/examples/proxy/proxy.py @@ -1,153 +1,50 @@ -import argparse -import atexit -import logging -import sys -import urllib.parse +#!/usr/bin/env python3 -import requests -from flask import Flask, Response, request -from waitress import serve -from zrok.model import ShareRequest, Share -from zrok.overview import EnvironmentAndResources, Overview +""" +Example of using zrok's proxy facility to create an HTTP proxy server. + +This example demonstrates how to: +1. Create a proxy share (optionally with a unique name for persistence) +2. Handle HTTP requests/responses through the proxy +3. Automatically clean up non-reserved shares on exit +""" + +import argparse +import logging import zrok +from zrok.proxy import ProxyShare # Setup logging logging.basicConfig( level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s' + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) logger = logging.getLogger(__name__) -app = Flask(__name__) -target_url = None -zrok_opts = {} -bindPort = 18081 -# List of hop-by-hop headers that should not be returned to the viewer -HOP_BY_HOP_HEADERS = { - 'connection', - 'keep-alive', - 'proxy-authenticate', - 'proxy-authorization', - 'te', - 'trailers', - 'transfer-encoding', - 'upgrade' -} - - -@app.route('/', defaults={'path': ''}, methods=['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS', 'PATCH']) -@app.route('/', methods=['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS', 'PATCH']) -def proxy(path): - global target_url - logger.info(f"Incoming {request.method} request to {request.path}") - logger.info(f"Headers: {dict(request.headers)}") - - # Forward the request to target URL - full_url = urllib.parse.urljoin(target_url, request.path) - logger.info(f"Forwarding to: {full_url}") - - # Copy request headers, excluding hop-by-hop headers - headers = {k: v for k, v in request.headers.items() if k.lower() not in HOP_BY_HOP_HEADERS and k.lower() != 'host'} - - try: - response = requests.request( - method=request.method, - url=full_url, - headers=headers, - data=request.get_data(), - stream=True - ) - - logger.info(f"Response status: {response.status_code}") - logger.info(f"Response headers: {dict(response.headers)}") - - # Filter out hop-by-hop headers from the response - filtered_headers = {k: v for k, v in response.headers.items() if k.lower() not in HOP_BY_HOP_HEADERS} - - return Response( - response.iter_content(chunk_size=8192), - status=response.status_code, - headers=filtered_headers - ) - - except Exception as e: - logger.error(f"Proxy error: {str(e)}", exc_info=True) - return str(e), 502 - - -@zrok.decor.zrok(opts=zrok_opts) -def run_proxy(): - # the port is only used to integrate zrok with frameworks that expect a "hostname:port" combo - serve(app, port=bindPort) - - -if __name__ == '__main__': +def main(): + """Main entry point.""" parser = argparse.ArgumentParser(description='Start a zrok proxy server') parser.add_argument('target_url', help='Target URL to proxy requests to') parser.add_argument('-n', '--unique-name', help='Unique name for the proxy instance') args = parser.parse_args() - target_url = args.target_url logger.info("=== Starting proxy server ===") - logger.info(f"Target URL: {target_url}") - logger.info(f"Logging level: {logger.getEffectiveLevel()}") + logger.info(f"Target URL: {args.target_url}") + # Load environment and create proxy share root = zrok.environment.root.Load() - my_env = EnvironmentAndResources( - environment=None, - shares=[] + proxy_share = ProxyShare.create( + root=root, + target=args.target_url, + unique_name=args.unique_name ) - overview = Overview.create(root=root) - for env_stuff in overview.environments: - if env_stuff.environment.z_id == root.env.ZitiIdentity: - my_env = EnvironmentAndResources( - environment=env_stuff.environment, - shares=env_stuff.shares - ) - break + + # Log access information and start the proxy + logger.info(f"Access proxy at: {', '.join(proxy_share.endpoints)}") + proxy_share.run() - if my_env: - logger.debug( - f"Found environment in overview with Ziti identity " - f"matching local environment: {my_env.environment.z_id}" - ) - else: - logger.error("No matching environment found") - sys.exit(1) - existing_reserved_share = None - for share in my_env.shares: - if share.token == args.unique_name: - existing_reserved_share = share - break - - if existing_reserved_share: - logger.debug(f"Found existing share with token: {existing_reserved_share.token}") - shr = Share(Token=existing_reserved_share.token, FrontendEndpoints=[existing_reserved_share.frontend_endpoint]) - else: - logger.debug(f"No existing share found with token: {args.unique_name}") - share_request = ShareRequest( - BackendMode=zrok.model.PROXY_BACKEND_MODE, - ShareMode=zrok.model.PUBLIC_SHARE_MODE, - Frontends=['public'], - Target="http-proxy", - Reserved=True - ) - if args.unique_name: - share_request.UniqueName = args.unique_name - - shr = zrok.share.CreateShare(root=root, request=share_request) - - def cleanup(): - zrok.share.ReleaseReservedShare(root=root, shr=shr) - logger.info(f"Share {shr.Token} released") - if not args.unique_name: - atexit.register(cleanup) - - zrok_opts['cfg'] = zrok.decor.Opts(root=root, shrToken=shr.Token, bindPort=bindPort) - - logger.info(f"Access proxy at: {', '.join(shr.FrontendEndpoints)}") - - run_proxy() +if __name__ == '__main__': + main() diff --git a/sdk/python/sdk/zrok/zrok/README.md b/sdk/python/sdk/zrok/zrok/README.md new file mode 100644 index 00000000..76b5fe8f --- /dev/null +++ b/sdk/python/sdk/zrok/zrok/README.md @@ -0,0 +1,47 @@ +# Zrok Python SDK + +## Proxy Facility + +The SDK includes a proxy facility that makes it easy to create and manage proxy shares. This is particularly useful when you need to: + +1. Create an HTTP proxy with zrok +2. Optionally reserve the proxy with a unique name for persistence +3. Automatically handle cleanup of non-reserved shares + +### Basic Usage + +```python +from zrok.proxy import ProxyShare +import zrok + +# Load the environment +root = zrok.environment.root.Load() + +# Create a temporary proxy share (will be cleaned up on exit) +proxy = ProxyShare.create(root=root, target="http://my-target-service") + +# Access the proxy's endpoints and token +print(f"Access proxy at: {proxy.endpoints}") +print(f"Share token: {proxy.token}") +``` + +### Creating a Reserved Proxy Share + +To create a proxy share that persists and can be reused: + +```python +# Create/retrieve a reserved proxy share with a unique name +proxy = ProxyShare.create( + root=root, + target="http://my-target-service", + unique_name="my-persistent-proxy" +) +``` + +When a `unique_name` is provided: + +1. If the zrok environment already has a share with that name, it will be reused +2. If no share exists, a new reserved share will be created +3. The share will be automatically cleaned up on exit if no `unique_name` is provided + +When a `unique_name` is not provided, the randomly generated share will be cleaned up on exit. diff --git a/sdk/python/sdk/zrok/zrok/proxy.py b/sdk/python/sdk/zrok/zrok/proxy.py new file mode 100644 index 00000000..65e2306b --- /dev/null +++ b/sdk/python/sdk/zrok/zrok/proxy.py @@ -0,0 +1,178 @@ +""" +Proxy share management functionality for the zrok SDK. +""" + +import atexit +import logging +import urllib.parse +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +import requests +from flask import Flask, Response, request +from waitress import serve +from zrok.environment.root import Root +from zrok.model import (PROXY_BACKEND_MODE, PUBLIC_SHARE_MODE, Share, + ShareRequest) +from zrok.overview import Overview +from zrok.share import CreateShare, ReleaseReservedShare + +import zrok + +logger = logging.getLogger(__name__) + +# List of hop-by-hop headers that should not be returned to the viewer +HOP_BY_HOP_HEADERS = { + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailers', + 'transfer-encoding', + 'upgrade' +} + +# The proxy only listens on the zrok socket, the port used to initialize the Waitress server is not actually bound or +# listening +DUMMY_PORT = 18081 + + +@dataclass +class ProxyShare: + """Represents a proxy share with its configuration and state.""" + root: Root + share: Share + target: str + unique_name: Optional[str] = None + _cleanup_registered: bool = False + _app: Optional[Flask] = None + + @classmethod + def create(cls, root: Root, target: str, unique_name: Optional[str] = None) -> 'ProxyShare': + """ + Create a new proxy share, handling reservation and cleanup logic based on unique_name. + + Args: + root: The zrok root environment + target: Target URL or service to proxy to + unique_name: Optional unique name for a reserved share + + Returns: + ProxyShare instance configured with the created share + """ + # First check if we have an existing reserved share with this name + if unique_name: + existing_share = cls._find_existing_share(root, unique_name) + if existing_share: + logger.debug(f"Found existing share with token: {existing_share.Token}") + return cls( + root=root, + share=existing_share, + target=target, + unique_name=unique_name + ) + + # Create new share request + share_request = ShareRequest( + BackendMode=PROXY_BACKEND_MODE, + ShareMode=PUBLIC_SHARE_MODE, + Target="http-proxy", + Frontends=['public'], + Reserved=bool(unique_name) + ) + if unique_name: + share_request.UniqueName = unique_name + + # Create the share + share = CreateShare(root=root, request=share_request) + logger.info(f"Created new proxy share with endpoints: {', '.join(share.FrontendEndpoints)}") + + # Create instance and setup cleanup if needed + instance = cls( + root=root, + share=share, + target=target, + unique_name=unique_name + ) + if not unique_name: + instance.register_cleanup() + return instance + + @staticmethod + def _find_existing_share(root: Root, unique_name: str) -> Optional[Share]: + """Find an existing share with the given unique name.""" + overview = Overview.create(root=root) + for env in overview.environments: + if env.environment.z_id == root.env.ZitiIdentity: + for share in env.shares: + if share.token == unique_name: + return Share(Token=share.token, FrontendEndpoints=[share.frontend_endpoint]) + return None + + def register_cleanup(self): + """Register cleanup handler to release the share on exit.""" + if not self._cleanup_registered: + def cleanup(): + ReleaseReservedShare(root=self.root, shr=self.share) + logger.info(f"Share {self.share.Token} released") + atexit.register(cleanup) + self._cleanup_registered = True + + def _create_app(self) -> Flask: + """Create and configure the Flask app for proxying.""" + app = Flask(__name__) + + @app.route('/', defaults={'path': ''}, methods=['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS']) + @app.route('/', methods=['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS']) + def proxy(path): + # Construct the target URL + url = urllib.parse.urljoin(self.target, path) + + # Forward the request + resp = requests.request( + method=request.method, + url=url, + headers={key: value for (key, value) in request.headers + if key.lower() not in HOP_BY_HOP_HEADERS}, + data=request.get_data(), + cookies=request.cookies, + allow_redirects=False, + stream=True + ) + + # Create the response + excluded_headers = HOP_BY_HOP_HEADERS.union({'host'}) + headers = [(name, value) for (name, value) in resp.raw.headers.items() + if name.lower() not in excluded_headers] + + return Response( + resp.iter_content(chunk_size=10*1024), + status=resp.status_code, + headers=headers + ) + return app + + def run(self): + """Start the proxy server.""" + if self._app is None: + self._app = self._create_app() + + # Create options dictionary for zrok decorator + zrok_opts: Dict[str, Any] = {} + zrok_opts['cfg'] = zrok.decor.Opts(root=self.root, shrToken=self.token, bindPort=DUMMY_PORT) + + @zrok.decor.zrok(opts=zrok_opts) + def run_server(): + serve(self._app, port=DUMMY_PORT) + run_server() + + @property + def endpoints(self) -> List[str]: + """Get the frontend endpoints for this share.""" + return self.share.FrontendEndpoints + + @property + def token(self) -> str: + """Get the share token.""" + return self.share.Token From 6d4cc020ce549903694dedf139ebe960cc10b5dd Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jan 2025 18:53:14 -0500 Subject: [PATCH 15/23] tidy py sdk --- sdk/python/examples/proxy/proxy.py | 2 +- sdk/python/sdk/zrok/zrok/proxy.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/python/examples/proxy/proxy.py b/sdk/python/examples/proxy/proxy.py index 94f3f010..d20597a4 100644 --- a/sdk/python/examples/proxy/proxy.py +++ b/sdk/python/examples/proxy/proxy.py @@ -40,7 +40,7 @@ def main(): target=args.target_url, unique_name=args.unique_name ) - + # Log access information and start the proxy logger.info(f"Access proxy at: {', '.join(proxy_share.endpoints)}") proxy_share.run() diff --git a/sdk/python/sdk/zrok/zrok/proxy.py b/sdk/python/sdk/zrok/zrok/proxy.py index 65e2306b..eb5b0f7d 100644 --- a/sdk/python/sdk/zrok/zrok/proxy.py +++ b/sdk/python/sdk/zrok/zrok/proxy.py @@ -128,13 +128,13 @@ class ProxyShare: def proxy(path): # Construct the target URL url = urllib.parse.urljoin(self.target, path) - + # Forward the request resp = requests.request( method=request.method, url=url, - headers={key: value for (key, value) in request.headers - if key.lower() not in HOP_BY_HOP_HEADERS}, + headers={key: value for (key, value) in request.headers + if key.lower() not in HOP_BY_HOP_HEADERS}, data=request.get_data(), cookies=request.cookies, allow_redirects=False, @@ -144,7 +144,7 @@ class ProxyShare: # Create the response excluded_headers = HOP_BY_HOP_HEADERS.union({'host'}) headers = [(name, value) for (name, value) in resp.raw.headers.items() - if name.lower() not in excluded_headers] + if name.lower() not in excluded_headers] return Response( resp.iter_content(chunk_size=10*1024), From 9294981305f2c20bf9fb093602dc5c91072b43f5 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 29 Jan 2025 03:24:56 -0500 Subject: [PATCH 16/23] support custom domains and insecure https targets --- sdk/python/examples/proxy/proxy.py | 7 +++- sdk/python/sdk/zrok/zrok/proxy.py | 54 ++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/sdk/python/examples/proxy/proxy.py b/sdk/python/examples/proxy/proxy.py index d20597a4..815464cd 100644 --- a/sdk/python/examples/proxy/proxy.py +++ b/sdk/python/examples/proxy/proxy.py @@ -28,6 +28,9 @@ def main(): parser = argparse.ArgumentParser(description='Start a zrok proxy server') parser.add_argument('target_url', help='Target URL to proxy requests to') parser.add_argument('-n', '--unique-name', help='Unique name for the proxy instance') + parser.add_argument('-f', '--frontends', nargs='+', help='One or more space-separated frontends to use') + parser.add_argument('-k', '--insecure', action='store_false', dest='verify_ssl', default=True, + help='Skip SSL verification') args = parser.parse_args() logger.info("=== Starting proxy server ===") @@ -38,7 +41,9 @@ def main(): proxy_share = ProxyShare.create( root=root, target=args.target_url, - unique_name=args.unique_name + unique_name=args.unique_name, + frontends=args.frontends, + verify_ssl=args.verify_ssl ) # Log access information and start the proxy diff --git a/sdk/python/sdk/zrok/zrok/proxy.py b/sdk/python/sdk/zrok/zrok/proxy.py index eb5b0f7d..a25e1293 100644 --- a/sdk/python/sdk/zrok/zrok/proxy.py +++ b/sdk/python/sdk/zrok/zrok/proxy.py @@ -47,9 +47,11 @@ class ProxyShare: unique_name: Optional[str] = None _cleanup_registered: bool = False _app: Optional[Flask] = None + verify_ssl: bool = True @classmethod - def create(cls, root: Root, target: str, unique_name: Optional[str] = None) -> 'ProxyShare': + def create(cls, root: Root, target: str, unique_name: Optional[str] = None, + frontends: Optional[List[str]] = None, verify_ssl: bool = True) -> 'ProxyShare': """ Create a new proxy share, handling reservation and cleanup logic based on unique_name. @@ -57,6 +59,8 @@ class ProxyShare: root: The zrok root environment target: Target URL or service to proxy to unique_name: Optional unique name for a reserved share + frontends: Optional list of frontends to use, takes precedence over root's default_frontend + verify_ssl: Whether to verify SSL certificates when forwarding requests. Returns: ProxyShare instance configured with the created share @@ -70,30 +74,39 @@ class ProxyShare: root=root, share=existing_share, target=target, - unique_name=unique_name + unique_name=unique_name, + verify_ssl=verify_ssl ) - # Create new share request + # Compose the share request + if frontends: + share_frontends = frontends + elif root.cfg and root.cfg.DefaultFrontend: + share_frontends = [root.cfg.DefaultFrontend] + else: + share_frontends = ['public'] + share_request = ShareRequest( BackendMode=PROXY_BACKEND_MODE, ShareMode=PUBLIC_SHARE_MODE, - Target="http-proxy", - Frontends=['public'], - Reserved=bool(unique_name) + Target=target, + Frontends=share_frontends, + Reserved=True ) if unique_name: share_request.UniqueName = unique_name # Create the share share = CreateShare(root=root, request=share_request) - logger.info(f"Created new proxy share with endpoints: {', '.join(share.FrontendEndpoints)}") + logger.debug(f"Created new proxy share with endpoints: {', '.join(share.FrontendEndpoints)}") - # Create instance and setup cleanup if needed + # Create class instance and setup cleanup-at-exit if we reserved a random share token instance = cls( root=root, share=share, target=target, - unique_name=unique_name + unique_name=unique_name, + verify_ssl=verify_ssl ) if not unique_name: instance.register_cleanup() @@ -111,13 +124,19 @@ class ProxyShare: return None def register_cleanup(self): - """Register cleanup handler to release the share on exit.""" + """Register cleanup handler to release randomly generated shares on exit.""" if not self._cleanup_registered: def cleanup(): - ReleaseReservedShare(root=self.root, shr=self.share) - logger.info(f"Share {self.share.Token} released") + try: + ReleaseReservedShare(root=self.root, shr=self.share) + logger.info(f"Share {self.share.Token} released") + except Exception as e: + logger.error(f"Error during cleanup: {e}") + + # Register for normal exit only atexit.register(cleanup) self._cleanup_registered = True + return cleanup # Return the cleanup function for reuse def _create_app(self) -> Flask: """Create and configure the Flask app for proxying.""" @@ -138,7 +157,8 @@ class ProxyShare: data=request.get_data(), cookies=request.cookies, allow_redirects=False, - stream=True + stream=True, + verify=self.verify_ssl ) # Create the response @@ -154,17 +174,17 @@ class ProxyShare: return app def run(self): - """Start the proxy server.""" - if self._app is None: + """Run the proxy server.""" + if not self._app: self._app = self._create_app() - # Create options dictionary for zrok decorator zrok_opts: Dict[str, Any] = {} zrok_opts['cfg'] = zrok.decor.Opts(root=self.root, shrToken=self.token, bindPort=DUMMY_PORT) @zrok.decor.zrok(opts=zrok_opts) def run_server(): - serve(self._app, port=DUMMY_PORT) + serve(self._app, port=DUMMY_PORT, _quiet=True) + run_server() @property From 2f31df47275a00e18a18a445ac0c3a04443b86c3 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 29 Jan 2025 03:25:45 -0500 Subject: [PATCH 17/23] implement default frontend config --- sdk/python/sdk/zrok/zrok/environment/root.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/python/sdk/zrok/zrok/environment/root.py b/sdk/python/sdk/zrok/zrok/environment/root.py index 65e483cb..ca9b93e2 100644 --- a/sdk/python/sdk/zrok/zrok/environment/root.py +++ b/sdk/python/sdk/zrok/zrok/environment/root.py @@ -19,6 +19,7 @@ class Metadata: @dataclass class Config: ApiEndpoint: str = "" + DefaultFrontend: str = "" @dataclass @@ -137,7 +138,10 @@ def __loadConfig() -> Config: cf = configFile() with open(cf) as f: data = json.load(f) - return Config(ApiEndpoint=data["api_endpoint"]) + return Config( + ApiEndpoint=data["api_endpoint"], + DefaultFrontend=data["default_frontend"] + ) def isEnabled() -> bool: From 41ce0809ab49b8cab232b23cab0795bd09bda9d6 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 29 Jan 2025 03:41:40 -0500 Subject: [PATCH 18/23] document the proxy example --- sdk/python/examples/proxy/README.md | 48 +++++++++++++++++++++++++++++ sdk/python/sdk/zrok/zrok/README.md | 47 ---------------------------- 2 files changed, 48 insertions(+), 47 deletions(-) create mode 100644 sdk/python/examples/proxy/README.md delete mode 100644 sdk/python/sdk/zrok/zrok/README.md diff --git a/sdk/python/examples/proxy/README.md b/sdk/python/examples/proxy/README.md new file mode 100644 index 00000000..a5fff84f --- /dev/null +++ b/sdk/python/examples/proxy/README.md @@ -0,0 +1,48 @@ + +# zrok Python Proxy Example + +This demonstrates using the ProxyShare class to forward requests from the public frontend to a target URL. + +## Run the Example + +```bash +LOG_LEVEL=INFO python ./proxy.py http://127.0.0.1:3000 +``` + +Expected output: + +```txt +2025-01-29 06:37:00,884 - __main__ - INFO - === Starting proxy server === +2025-01-29 06:37:00,884 - __main__ - INFO - Target URL: http://127.0.0.1:3000 +2025-01-29 06:37:01,252 - __main__ - INFO - Access proxy at: https://24x0pq7s6jr0.zrok.example.com:443 +2025-01-29 06:37:07,981 - zrok.proxy - INFO - Share 24x0pq7s6jr0 released +``` + +## Basic Usage + +```python +from zrok.proxy import ProxyShare +import zrok + +# Load the environment +root = zrok.environment.root.Load() + +# Create a temporary proxy share (will be cleaned up on exit) +proxy = ProxyShare.create(root=root, target="http://my-target-service") + +# Access the proxy's endpoints and token +print(f"Access proxy at: {proxy.endpoints}") +proxy.run() +``` + +## Creating a Reserved Proxy Share + +To create a share token that persists and can be reused, run the example `proxy.py --unique-name my-persistent-proxy`. If the unique name already exists it will be reused. Here's how it works: + +```python +proxy = ProxyShare.create( + root=root, + target="http://127.0.0.1:3000", + unique_name="my-persistent-proxy" +) +``` diff --git a/sdk/python/sdk/zrok/zrok/README.md b/sdk/python/sdk/zrok/zrok/README.md deleted file mode 100644 index 76b5fe8f..00000000 --- a/sdk/python/sdk/zrok/zrok/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Zrok Python SDK - -## Proxy Facility - -The SDK includes a proxy facility that makes it easy to create and manage proxy shares. This is particularly useful when you need to: - -1. Create an HTTP proxy with zrok -2. Optionally reserve the proxy with a unique name for persistence -3. Automatically handle cleanup of non-reserved shares - -### Basic Usage - -```python -from zrok.proxy import ProxyShare -import zrok - -# Load the environment -root = zrok.environment.root.Load() - -# Create a temporary proxy share (will be cleaned up on exit) -proxy = ProxyShare.create(root=root, target="http://my-target-service") - -# Access the proxy's endpoints and token -print(f"Access proxy at: {proxy.endpoints}") -print(f"Share token: {proxy.token}") -``` - -### Creating a Reserved Proxy Share - -To create a proxy share that persists and can be reused: - -```python -# Create/retrieve a reserved proxy share with a unique name -proxy = ProxyShare.create( - root=root, - target="http://my-target-service", - unique_name="my-persistent-proxy" -) -``` - -When a `unique_name` is provided: - -1. If the zrok environment already has a share with that name, it will be reused -2. If no share exists, a new reserved share will be created -3. The share will be automatically cleaned up on exit if no `unique_name` is provided - -When a `unique_name` is not provided, the randomly generated share will be cleaned up on exit. From 5c64a73aad0b47fca0e4d91de394c0be8cea5e5a Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 29 Jan 2025 04:06:37 -0500 Subject: [PATCH 19/23] add share mode --- sdk/python/examples/proxy/proxy.py | 9 ++++++++- sdk/python/sdk/zrok/zrok/proxy.py | 25 +++++++++++++++---------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/sdk/python/examples/proxy/proxy.py b/sdk/python/examples/proxy/proxy.py index 815464cd..a0462971 100644 --- a/sdk/python/examples/proxy/proxy.py +++ b/sdk/python/examples/proxy/proxy.py @@ -31,23 +31,30 @@ def main(): parser.add_argument('-f', '--frontends', nargs='+', help='One or more space-separated frontends to use') parser.add_argument('-k', '--insecure', action='store_false', dest='verify_ssl', default=True, help='Skip SSL verification') + parser.add_argument('-s', '--share-mode', default='public', choices=['public', 'private'], + help='Share mode (default: public)') args = parser.parse_args() logger.info("=== Starting proxy server ===") logger.info(f"Target URL: {args.target_url}") + logger.info(f"Share mode: {args.share_mode}") # Load environment and create proxy share root = zrok.environment.root.Load() proxy_share = ProxyShare.create( root=root, target=args.target_url, + share_mode=args.share_mode, unique_name=args.unique_name, frontends=args.frontends, verify_ssl=args.verify_ssl ) # Log access information and start the proxy - logger.info(f"Access proxy at: {', '.join(proxy_share.endpoints)}") + if args.share_mode == "public": + logger.info(f"Access proxy at: {', '.join(proxy_share.endpoints)}") + elif args.share_mode == "private": + logger.info(f"Run a private access frontend: 'zrok access private {proxy_share.token}'") proxy_share.run() diff --git a/sdk/python/sdk/zrok/zrok/proxy.py b/sdk/python/sdk/zrok/zrok/proxy.py index a25e1293..a1244ca3 100644 --- a/sdk/python/sdk/zrok/zrok/proxy.py +++ b/sdk/python/sdk/zrok/zrok/proxy.py @@ -12,7 +12,7 @@ import requests from flask import Flask, Response, request from waitress import serve from zrok.environment.root import Root -from zrok.model import (PROXY_BACKEND_MODE, PUBLIC_SHARE_MODE, Share, +from zrok.model import (PROXY_BACKEND_MODE, PUBLIC_SHARE_MODE, PRIVATE_SHARE_MODE, Share, ShareRequest) from zrok.overview import Overview from zrok.share import CreateShare, ReleaseReservedShare @@ -50,7 +50,7 @@ class ProxyShare: verify_ssl: bool = True @classmethod - def create(cls, root: Root, target: str, unique_name: Optional[str] = None, + def create(cls, root: Root, target: str, share_mode: str = PUBLIC_SHARE_MODE, unique_name: Optional[str] = None, frontends: Optional[List[str]] = None, verify_ssl: bool = True) -> 'ProxyShare': """ Create a new proxy share, handling reservation and cleanup logic based on unique_name. @@ -79,16 +79,18 @@ class ProxyShare: ) # Compose the share request - if frontends: - share_frontends = frontends - elif root.cfg and root.cfg.DefaultFrontend: - share_frontends = [root.cfg.DefaultFrontend] - else: - share_frontends = ['public'] + share_frontends = [] + if share_mode == PUBLIC_SHARE_MODE: + if frontends: + share_frontends = frontends + elif root.cfg and root.cfg.DefaultFrontend: + share_frontends = [root.cfg.DefaultFrontend] + else: + share_frontends = ['public'] share_request = ShareRequest( BackendMode=PROXY_BACKEND_MODE, - ShareMode=PUBLIC_SHARE_MODE, + ShareMode=share_mode, Target=target, Frontends=share_frontends, Reserved=True @@ -98,7 +100,10 @@ class ProxyShare: # Create the share share = CreateShare(root=root, request=share_request) - logger.debug(f"Created new proxy share with endpoints: {', '.join(share.FrontendEndpoints)}") + if share_mode == PUBLIC_SHARE_MODE: + logger.debug(f"Created new proxy share with endpoints: {', '.join(share.FrontendEndpoints)}") + elif share_mode == PRIVATE_SHARE_MODE: + logger.debug(f"Created new private share with token: {share.Token}") # Create class instance and setup cleanup-at-exit if we reserved a random share token instance = cls( From c471aefe7a8c3dfce5883726a5b16d7d997e9142 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 29 Jan 2025 04:33:24 -0500 Subject: [PATCH 20/23] add Jupyter notebook proxy example --- CHANGELOG.md | 5 ++ sdk/python/examples/proxy/proxy.ipynb | 100 ++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 sdk/python/examples/proxy/proxy.ipynb diff --git a/CHANGELOG.md b/CHANGELOG.md index afa20c85..013e23c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ CHANGE: Add usage hint in `zrok config get --help` to clarify how to list all valid `configName` and their current values by running `zrok status`. +CHANGE: The Python SDK's `Overview()` function was refactored as a class method (https://github.com/openziti/zrok/pull/846). + +FEATURE: The Python SDK now includes a `ProxyShare` class providing an HTTP proxy for public and private shares and a + Jupyter notebook example (https://github.com/openziti/zrok/pull/847). + ## v0.4.46 FEATURE: Linux service template for systemd user units (https://github.com/openziti/zrok/pull/818) diff --git a/sdk/python/examples/proxy/proxy.ipynb b/sdk/python/examples/proxy/proxy.ipynb new file mode 100644 index 00000000..2fb5977a --- /dev/null +++ b/sdk/python/examples/proxy/proxy.ipynb @@ -0,0 +1,100 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "52d42237", + "metadata": {}, + "outputs": [], + "source": [ + "! pip install zrok" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0a33915c", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import zrok\n", + "from zrok.proxy import ProxyShare\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0db6b615", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "target_url = \"http://127.0.0.1:8000/\"\n", + "unique_name = \"myuniquename\" # a name to reuse each run or 'None' for random\n", + "share_mode = \"public\" # \"public\" or \"private\"\n", + "frontend = \"public\" # custom domain frontend or \"public\"\n", + "\n", + "if unique_name.lower() == \"none\":\n", + " unique_name = None\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3efcfa5", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "zrok_env = zrok.environment.root.Load() # Load the environment from ~/.zrok\n", + "\n", + "proxy_share = ProxyShare.create(\n", + " root=zrok_env,\n", + " target=target_url,\n", + " frontends=[frontend],\n", + " share_mode=share_mode,\n", + " unique_name=unique_name,\n", + " verify_ssl=True # Set 'False' to skip SSL verification\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21966557", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "if share_mode == \"public\":\n", + " print(f\"Access proxy at: {', '.join(proxy_share.endpoints)}\")\n", + "elif share_mode == \"private\":\n", + " print(f\"Run a private access frontend: 'zrok access private {proxy_share.token}'\")\n", + "\n", + "proxy_share.run()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From dfbaae37ae0ade7a934d7773124a2a7e92ddb9f5 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 29 Jan 2025 04:51:14 -0500 Subject: [PATCH 21/23] publish to pypi in a separate job if testpypi succeeds --- .github/workflows/build-wheels.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 0fe9a743..7dce021c 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -45,7 +45,7 @@ jobs: name: zrok_sdk_${{ matrix.spec.target }} path: ${{ github.workspace }}/sdk/python/sdk/zrok/dist/* - publish: + publish-testpypi: runs-on: ubuntu-20.04 needs: [ build_wheels ] permissions: @@ -72,6 +72,19 @@ jobs: skip-existing: true verbose: true + publish-pypi: + runs-on: ubuntu-20.04 + needs: [ publish-testpypi ] + permissions: + id-token: write + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: ./download + merge-multiple: true + pattern: zrok_sdk_* + - name: Publish wheels (PyPI) uses: pypa/gh-action-pypi-publish@release/v1 with: From ed98590dc385693f59f868d5c06007f78acde379 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 29 Jan 2025 05:18:19 -0500 Subject: [PATCH 22/23] fix wheels download dir --- .github/workflows/build-wheels.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 7dce021c..25440811 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -54,16 +54,10 @@ jobs: - name: Download artifacts uses: actions/download-artifact@v4 with: - path: ./download + path: ./dist merge-multiple: true pattern: zrok_sdk_* - - name: check - run: | - ls -lR ./download/ - mkdir dist - cp ./download/* ./dist/ - - name: Publish wheels (TestPYPI) uses: pypa/gh-action-pypi-publish@release/v1 with: @@ -81,7 +75,7 @@ jobs: - name: Download artifacts uses: actions/download-artifact@v4 with: - path: ./download + path: ./dist merge-multiple: true pattern: zrok_sdk_* From 7f10f1ced7c6b1de37cf2fbab81b12fcd4509d61 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 29 Jan 2025 05:41:56 -0500 Subject: [PATCH 23/23] announce pypi publish fix --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2933da2d..0d868937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## v0.4.47 + +FIX: PyPi publishing was failing due to a CI issue (https://github.com/openziti/zrok/issues/849) + ## v0.4.46 FEATURE: Linux service template for systemd user units (https://github.com/openziti/zrok/pull/818)