chore: run the key server in dev backend

We can use mitmproxy to terminate tls! the best way is to run a second
mitmproxy, like we run a second nginx, and feed it the certs generated
by the key-server. We need a little stuff in the key-server to support
this because mitmproxy wants the whole bundle of tls cert, tls key, and
ca cert in a single PEM (one PEM can contain multiple entities) and then
we can "notify" of configuration changes by touching the main mitmproxy
script since it has hot module reload. Probably. Nobody's going to run
their dev backend instance for 3 years anyway.

Closes EXEC-2524

Author: sfoster1

.gitignore

@@ -160,3 +160,4 @@ api/docs/dist/v2/
 package-testing/results
 all_analysis_results/
 auth-server/.env
+key-server/.key-server-storage

Makefile

@@ -349,7 +349,10 @@ dev-backend-flex:
 		$(MAKE) -C auth-server dev ';' \
 		$(MAKE) -C robot-server dev-flex OT_ROBOT_SERVER_auth_server_url=http://localhost:31950 BEHIND_DEV_PROXY=1 ';' \
 		$(MAKE) -C system-server dev OT_SYSTEM_SERVER_auth_server_url=http://localhost:31950 ';' \
-		$(MAKE) dev-proxy
+		$(MAKE) -C key-server dev-mitmproxy ';' \
+		$(MAKE) dev-proxy ';' \
+		$(MAKE) dev-proxy-tls
+
 
 # Assuming our dev servers are running separately (make -C robot-server dev, make -C auth-server dev, etc.),
 # this sets up a reverse proxy that listens on localhost:31950 and forwards each request
@@ -367,3 +370,12 @@ dev-proxy:
 	    --mode reverse:http://localhost:2@31950 \
 	    --set connection_strategy=lazy \
 	    --script scripts/dev_proxy.py
+
+.PHONY: dev-proxy-tls
+dev-proxy-tls:
+	sleep 1
+	$(UV) tool run --python $(UV_PYTHON) --from mitmproxy==12.2.1 mitmdump \
+		--mode reverse:http://localhost:2@32313 \
+		--set connection_strategy=lazy \
+		--script scripts/dev_proxy.py \
+		--certs key-server/.key-server-storage/tls/flex-certs.pem

key-server/Makefile

@@ -42,7 +42,7 @@ clean_cmd = $(SHX) rm -rf build .coverage coverage.xml '*.egg-info'
 clean_cache_cmd = $(SHX) rm -rf '**/__pycache__' '**/*.pyc' '**/.mypy_cache'
 clean_wheel_cmd = $(clean_cmd) dist/*.whl
 clean_all_cmd = $(clean_cmd) dist
-dev_port ?= "33950"
+dev_port ?= "33960"
 
 .PHONY: all
 all: clean wheel
@@ -97,9 +97,16 @@ format:
 	$(ruff) format .
 	$(ruff) check --select I --fix .
 
+# the dev task is pretty good for running the server just locally for quick tests.
 .PHONY: dev
 dev:
-	OT_KEY_SERVER_dot_env_path=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))/dev.env $(python) -m key_server --port $(dev_port) --reload --log-level=debug
+	OT_KEY_SERVER_dot_env_path=dev.env $(python) -m key_server --port $(dev_port) --reload --log-level=debug
+
+# the dev-mitmproxy task integrates with the whole-backend mode that runs a reverse proxy in
+# front of all the servers. don't run it unless you're going to run mitmproxy
+.PHONY: dev-mitmproxy
+dev-mitmproxy:
+	OT_KEY_SERVER_dot_env_path=dev-mitmproxy.env $(python) -m key_server --port $(dev_port) --reload --log-level=debug
 
 # Note: No `push` target, since this project only runs on Flex (OT-3), not OT-2.
 

key-server/dev-mitmproxy.env

@@ -0,0 +1,4 @@
+OT_KEY_SERVER_tls_server_integration=dev-mitmproxy
+OT_KEY_SERVER_base_directory=.key-server-storage
+OT_KEY_SERVER_tls_directory=.key-server-storage/tls
+OT_KEY_SERVER_mitmproxy_touch_path=../scripts/dev_proxy.py

key-server/key_server/config/dependency.py

@@ -57,6 +57,11 @@ def resolve_config() -> ResolvedConfig:
         secure_volume_size_mb=base_config.secure_volume_size_mb,
         tls_directory=tls_base_dir,
         tls_server_integration=base_config.tls_server_integration,
+        mitmproxy_touch_path=(
+            base_config.mitmproxy_touch_path
+            if base_config.tls_server_integration == "dev-mitmproxy"
+            else None
+        ),
     )
 
 

key-server/key_server/config/models.py

@@ -57,9 +57,15 @@ class KeyServerConfig(BaseSettings):
         default="automatically_make_temporary",
         description="The location in which to store tls keys and certs for tls termination",
     )
-    tls_server_integration: Literal["systemd-nginx", "dev-none"] = Field(
-        description="How the server should notify a TLS termination layer a certificate has rotated.",
-        default="systemd-nginx",
+    tls_server_integration: Literal["systemd-nginx", "dev-none", "dev-mitmproxy"] = (
+        Field(
+            description="How the server should notify a TLS termination layer a certificate has rotated.",
+            default="systemd-nginx",
+        )
+    )
+    mitmproxy_touch_path: Path | None = Field(
+        default=None,
+        description="File to touch to notify mitmproxy it needs to rotate. Ignored if tls_server_integration is not dev-mitmproxy.",
     )
 
 
@@ -71,4 +77,5 @@ class ResolvedConfig(BaseModel):
     image_mount_point: Path
     secure_volume_size_mb: int
     tls_directory: Path
-    tls_server_integration: Literal["systemd-nginx", "dev-none"]
+    tls_server_integration: Literal["systemd-nginx", "dev-none", "dev-mitmproxy"]
+    mitmproxy_touch_path: Path | None

key-server/key_server/tls/cryptography_utils.py

@@ -220,3 +220,8 @@ def install_tls_cert(tls_cert_dir: Path, tls_cert: SignedCert) -> X509Pair:
     )
     tls_cert.keypath.unlink()
     return pair
+
+
+def make_pem_bundle(ca: X509Pair, tls: X509Pair, path: Path) -> None:
+    """Write a CA and TLS pair to a single file."""
+    file_utils.make_pem_bundle(ca.cert, tls.cert, tls.key, path)

key-server/key_server/tls/file_utils.py

@@ -264,3 +264,25 @@ def load_ca_cert(maybe_cert: Path) -> x509.Certificate | None:  # noqa: C901
         )
         return None
     return cert
+
+
+def make_pem_bundle(
+    ca_cert: x509.Certificate,
+    tls_cert: x509.Certificate,
+    tls_key: ec.EllipticCurvePrivateKey,
+    path: Path,
+) -> None:
+    """Write a CA cert, end entity cert, and end entity key to a single PEM for certain TLS terminators."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("wb") as combined_pem:
+        combined_pem.write(
+            tls_key.private_bytes(
+                serialization.Encoding.PEM,
+                serialization.PrivateFormat.PKCS8,
+                serialization.NoEncryption(),
+            )
+        )
+        combined_pem.write(b"\n")
+        combined_pem.write(tls_cert.public_bytes(serialization.Encoding.PEM))
+        combined_pem.write(b"\n")
+        combined_pem.write(ca_cert.public_bytes(serialization.Encoding.PEM))

key-server/key_server/tls/manager.py

@@ -4,9 +4,10 @@
 import logging
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import Final, Literal, Self, Type
+from typing import Awaitable, Callable, Final, Literal, Self, Type, cast
 
 from .ca_manager import TLSCAManager
+from .cryptography_utils import make_pem_bundle
 from .ee_manager import TLSEEManager
 from .robot_details.interface import RobotDetails
 
@@ -37,7 +38,8 @@ async def create(
         ca_cert_dir: Path,
         ca_key_dir: Path,
         tls_ee_dir: Path,
-        terminator_reload: Literal["systemd-nginx", "dev-none"],
+        terminator_reload: Literal["systemd-nginx", "dev-none", "dev-mitmproxy"],
+        mitmproxy_touch_path: Path | None = None,
     ) -> Self:
         """Create a TLS manager, taking several useful setup steps."""
         if terminator_reload == "systemd-nginx":
@@ -52,6 +54,12 @@ async def create(
 
         robot_hostname, robot_ips = await robot_details.get_details()
         ca_manager = TLSCAManager(key_dir=ca_key_dir, ca_cert_dir=ca_cert_dir)
+        mitm_rotator = _MITMProxyCertRotator(mitmproxy_touch_path)
+        rotators = {
+            "systemd-nginx": TLSEEManager.rotate_cert_nginx,
+            "dev-none": TLSEEManager.rotate_cert_none,
+            "dev-mitmproxy": cast(Callable[[], Awaitable[None]], mitm_rotator),
+        }
         ee_manager = TLSEEManager(
             key_dir=tls_ee_dir,
             ee_cert_dir=tls_ee_dir,
@@ -61,16 +69,14 @@ async def create(
             # the easiest way is to start with no IP addresses and then to let the system that configures
             # IP addresses tell us when they start to exist.
             robot_ips=robot_ips,
-            rotate_fn=(
-                TLSEEManager.rotate_cert_nginx
-                if terminator_reload == "systemd-nginx"
-                else TLSEEManager.rotate_cert_none
-            ),
+            rotate_fn=rotators[terminator_reload],
         )
 
         obj = cls(
             ca_manager=ca_manager, ee_manager=ee_manager, robot_details=robot_details
         )
+        if terminator_reload == "dev-mitmproxy":
+            mitm_rotator.set_manager(obj)
         await obj.refresh_ee()
         await obj.schedule_expiry_task(cls.EXPIRY_CHECKER_POLL_PERIOD)
         await obj.schedule_robot_details_task()
@@ -179,3 +185,26 @@ async def _robot_details_task_outer(self) -> None:
         except BaseException:
             LOG.exception("Robot details task failed")
             raise
+
+
+class _MITMProxyCertRotator:
+    def __init__(self, touch_path: Path | None) -> None:
+        self._manager: TLSManager | None = None
+        self._path = touch_path
+
+    def set_manager(self, manager: TLSManager) -> None:
+        self._manager = manager
+
+    async def __call__(self) -> None:
+        """Rotate and tell the MITMProxy about it."""
+        if not self._manager:
+            return
+        if not self._manager._ee_manager._ee_pair:
+            return
+        make_pem_bundle(
+            self._manager._ca_manager._current_ca,
+            self._manager._ee_manager._ee_pair,
+            self._manager._ee_manager._ee_cert_dir / "flex-certs.pem",
+        )
+        if self._path:
+            self._path.touch()