t3-code-android-nightly/.repos/alchemy-effect/packages/alchemy/test/Cloudflare/Container/Container.test.ts

import * as Cloudflare from "@/Cloudflare";
import * as Test from "@/Test/Alchemy";
import { destroy as destroyStack } from "@/Destroy.ts";
import { State, inMemoryState } from "@/State";
import * as Plan from "@/Plan.ts";
import { Stage } from "@/Stage.ts";
import * as Layer from "effect/Layer";
import * as Containers from "@distilled.cloud/cloudflare/containers";
import * as Cause from "effect/Cause";
import * as Exit from "effect/Exit";
import { assert, describe, expect } from "alchemy-test";
import AttachmentStack, {
  attachmentStack,
  stackName as attachmentStackName,
} from "./fixtures/attachment/stack.ts";
import * as Effect from "effect/Effect";
import { MinimumLogLevel } from "effect/References";
import * as Schedule from "effect/Schedule";
import * as HttpClient from "effect/http/HttpClient";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import makeAsyncContainerStack from "./fixtures/async/stack.ts";
import EffectfulStack from "./fixtures/effectful/stack.ts";
import makeExternalStack from "./fixtures/external/stack.ts";
import makeInferredClassStack from "./fixtures/inferred/stack.ts";
import { DEMO_PLAIN, DEMO_SECRET } from "./fixtures/remote/object.ts";
import makeRemoteStack from "./fixtures/remote/stack.ts";

describe.concurrent.each([
  {
    dev: true,
    stage: process.env.ALCHEMY_TEST_STAGE
      ? `${process.env.ALCHEMY_TEST_STAGE}-local`
      : "test-local",
    // Must cover the full readiness budget (readinessRetries × 3s ≈ 3 min)
    // when a saturated machine keeps the local container answering 500s.
    timeout: 300_000,
  },
  {
    dev: false,
    stage: process.env.ALCHEMY_TEST_STAGE ?? "test-live",
    timeout: 300_000,
  },
])(
  "Container (dev: $dev)",
  ({ dev, stage, timeout }) => {
    const state = dev ? inMemoryState() : Cloudflare.state();
    const AsyncContainerStack = makeAsyncContainerStack(state);
    const ExternalStack = makeExternalStack(state);
    const InferredClassStack = makeInferredClassStack(state);
    const RemoteStack = makeRemoteStack(state);

    // We need to create a new test context for each test case because the
    // local runner stops running after the root scope is closed, which happens
    // in an afterAll hook registered by `Test.make`.
    const make = (remote = false) =>
      Test.make({
        providers: Cloudflare.providers(),
        state: remote ? Cloudflare.state() : state,
        stage,
        dev,
      });

    const logLevel = Effect.provideService(
      MinimumLogLevel,
      process.env.DEBUG ? "Debug" : "Info",
    );

    // Container image build + push + worker/DO deploy comfortably exceeds the
    // default 120s hook budget, so give every deploy/destroy plenty of room.
    const HOOK_TIMEOUT = 600_000;

    /**
     * Effect-native container (`main`): the entrypoint Effect is bundled into a
     * generated image. The Durable Object proxies into the container two ways —
     * RPC (`container.ping()` / `container.readObject()`) and HTTP over its
     * port-3000 server (`getTcpPort(3000).fetch`). The bucket tests prove the
     * full end-to-end R2 path: a value written through the DO's NATIVE binding is
     * read back from inside the container over its scoped HTTP token, surfaced
     * both via RPC and via fetch.
     */
    describe(
      "effectful container (main)",
      { tags: ["provider:cloudflare:r2", "live"] },
      () => {
        const { test, beforeAll, afterAll, deploy, destroy } = make(true);
        const stack = beforeAll(deploy(EffectfulStack), {
          timeout: HOOK_TIMEOUT,
        });
        afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(EffectfulStack), {
          timeout: HOOK_TIMEOUT,
        });

        test(
          "RPC: ping round-trips into the container",
          Effect.gen(function* () {
            const { url } = yield* stack;

            const pong = yield* fetchReady(new URL("/ping", url), "pong");
            expect(pong).toContain("pong");
          }).pipe(logLevel),
          { timeout },
        );

        test(
          "fetch: serves over its TCP port",
          Effect.gen(function* () {
            const { url } = yield* stack;

            const hello = yield* fetchReady(
              new URL("/hello", url),
              "effectful container",
            );
            expect(hello).toContain("effectful container");
          }).pipe(logLevel),
          { timeout },
        );

        // A container is a real process — it has no workerd bindings, so a
        // capability's only channel into it is the binding contract's `env`.
        // The provider used to declare that contract and silently drop the
        // values; this pins that a `Binding.Service`'s `host.bind({ env })`
        // lands in the container's `process.env`, resolved Output and all.
        // Same mechanism `Prisma.Connect` rides into a container.
        test(
          "a Binding.Service's bound env reaches the container",
          Effect.gen(function* () {
            const { url, bucketName } = yield* stack;

            const body = yield* fetchReady(
              new URL("/bound-env", url),
              bucketName,
            );
            expect(JSON.parse(body)).toEqual({ value: bucketName });
          }).pipe(logLevel),
          { timeout },
        );

        test(
          "RPC: reads an R2 object from inside the container",
          Effect.gen(function* () {
            const { url } = yield* stack;

            yield* seed(url, "rpc.txt", "hello-rpc");
            const body = yield* fetchReady(
              new URL("/rpc?key=rpc.txt", url),
              "hello-rpc",
            );
            expect(body).toContain("hello-rpc");
          }).pipe(logLevel),
          { timeout },
        );

        test(
          "fetch: reads an R2 object from inside the container",
          Effect.gen(function* () {
            const { url } = yield* stack;

            yield* seed(url, "fetch.txt", "hello-fetch");
            const body = yield* fetchReady(
              new URL("/fetch?key=fetch.txt", url),
              "hello-fetch",
            );
            expect(body).toContain("hello-fetch");
          }).pipe(logLevel),
          { timeout },
        );
      },
    );

    /**
     * External container (`context` / `dockerfile`): Alchemy builds the user's
     * Dockerfile against the context directory (nginx serving a static page on
     * port 8080) and the DO proxies a request to it.
     */
    describe("external container (context/dockerfile)", () => {
      const { test, beforeAll, afterAll, deploy, destroy } = make();
      const stack = beforeAll(deploy(ExternalStack), { timeout: HOOK_TIMEOUT });
      afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(ExternalStack), {
        timeout: HOOK_TIMEOUT,
      });

      test(
        "builds the user Dockerfile and serves it over its TCP port",
        Effect.gen(function* () {
          const { url } = yield* stack;

          const hello = yield* fetchReady(
            new URL("/hello", url),
            "external container",
          );
          expect(hello).toContain("external container");
        }).pipe(logLevel),
        { tags: [...(dev ? ["local"] : ["live"])], timeout },
      );
    });

    /**
     * Remote container (`image`): Alchemy pulls a pre-built public image and
     * re-pushes it to Cloudflare's registry without building anything; the DO
     * proxies a request to it.
     */
    describe(
      "remote container (image)",
      { tags: ["provider:cloudflare:r2"] },
      () => {
        const { test, beforeAll, afterAll, deploy, destroy } = make();
        const stack = beforeAll(deploy(RemoteStack), { timeout: HOOK_TIMEOUT });
        afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(RemoteStack), {
          timeout: HOOK_TIMEOUT,
        });

        test(
          "preserves memoryMib and serves the remote image over its TCP port",
          Effect.gen(function* () {
            const { url, app } = yield* stack;

            expect(app.configuration.memoryMib).toBe(4096);
            expect(app.configuration.instanceType).toBeUndefined();
            expect(app.applicationId.startsWith("dev:")).toBe(dev);

            const hello = yield* fetchReady(new URL("/hello", url), "method");
            expect(hello).toContain("method");
          }).pipe(logLevel),
          { tags: [...(dev ? ["local"] : ["live"])], timeout },
        );

        // A pre-built image has no Effect runtime to inject config into, so `env`
        // is the only channel: it lands as the ContainerApplication's
        // `environmentVariables` and shows up as plain `process.env` entries
        // inside the image. Covers all three value shapes — a literal, a
        // `Redacted` (encrypted in state, plain in the container), and an Output
        // resolved from a sibling resource (the canonical shape of a database
        // connection string).
        test(
          "passes env — literal, Redacted, and sibling-resource Output — into the image",
          Effect.gen(function* () {
            const { url, bucketName } = yield* stack;
            const body = yield* fetchReady(
              new URL("/hello", url),
              "DEMO_PLAIN",
            );
            const echoed = JSON.parse(body) as { env: Record<string, string> };

            expect(echoed.env.DEMO_PLAIN).toBe(DEMO_PLAIN);
            expect(echoed.env.DEMO_SECRET).toBe(DEMO_SECRET);
            expect(echoed.env.DEMO_BUCKET).toBe(bucketName);
          }).pipe(logLevel),
          { tags: [...(dev ? ["local"] : ["live"])], timeout },
        );

        // The proxy pattern from #1334 (`yield* fetch(yield* HttpServerRequest)`):
        // worker → DO fetch handler → container port, forwarding the raw incoming
        // request. In the live variant the incoming web Request carries an
        // https:// URL and workerd rejects TLS on container ports ("Connecting to
        // a container using HTTPS is not currently supported"), so this pins the
        // runtime's scheme downgrade on the container hop; dev serves http:// and
        // just pins the passthrough itself.
        test(
          "proxies the raw incoming request through the DO to the container",
          Effect.gen(function* () {
            const { url } = yield* stack;

            // The echo image reflects the request as JSON ("method" only appears
            // in a real echo response, never in an error page).
            const body = yield* fetchReady(
              new URL("/passthrough", url),
              "method",
            );
            expect(body).toContain("method");
          }).pipe(logLevel),
          { tags: [...(dev ? ["local"] : ["live"])], timeout },
        );
      },
    );
    /**
     * Container bound directly on a plain async Worker's `env` (issue #953),
     * with the Container's logical id EQUAL to the env key
     * (`ECHO: Container("ECHO", …)`). The Worker's two halves — the
     * `durable_object_namespace` binding and the `containers` script metadata —
     * describe the same env entry, so they are contributed under one `sid`.
     * Contributing them under two (the env key and the ContainerApplication's
     * logical id) made them collide for exactly this shape, and binding rows
     * are collapsed by sid — last write wins — so the namespace binding was
     * silently dropped. Live, the Worker then uploads the Container declaration
     * itself as a `json` binding (`{"_id":"Effect","op":"alchemy/EffectClass"}`)
     * and `getContainer` dies with `t.idFromName is not a function`; in dev the
     * local runtime holds container metadata for a class it has no namespace
     * for and the deploy dies with
     * `Durable Object namespace AsyncEchoObject not found`.
     */
    describe("async container bound on env", () => {
      const { test, beforeAll, afterAll, deploy, destroy } = make();
      const stack = beforeAll(deploy(AsyncContainerStack), {
        timeout: HOOK_TIMEOUT,
      });
      afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(AsyncContainerStack), {
        timeout: HOOK_TIMEOUT,
      });

      test(
        "binds the container class as a durable object namespace",
        Effect.gen(function* () {
          const { url } = yield* stack;

          const body = yield* fetchReady(new URL("/binding", url), "kind");
          expect(JSON.parse(body)).toEqual({
            kind: "durable_object_namespace",
          });
        }).pipe(logLevel),
        { tags: [...(dev ? ["local"] : ["live"])], timeout },
      );

      test(
        "serves through the container-backed DO class",
        Effect.gen(function* () {
          const { url } = yield* stack;

          // The echo image reflects the request as JSON ("method" only appears
          // in a real echo response, never in an error page) — proof the
          // request went Worker → DO class → container port 8080 and back.
          const hello = yield* fetchReady(new URL("/hello", url), "method");
          expect(hello).toContain("method");
        }).pipe(logLevel),
        { tags: [...(dev ? ["local"] : ["live"])], timeout },
      );
    });
    /**
     * Issue #1321 — the same sid collision as above in the shape the
     * Containers guide documents: `className` omitted, so the env key, the
     * Container's logical id and the DO class are all one name
     * (`Probe: Container("Probe", …)`). Deployed and driven over HTTP: the
     * Worker must see a real namespace and reach the container through it.
     */
    describe("async container with inferred class name", () => {
      const { test, beforeAll, afterAll, deploy, destroy } = make();
      const stack = beforeAll(deploy(InferredClassStack), {
        timeout: HOOK_TIMEOUT,
      });
      afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(InferredClassStack), {
        timeout: HOOK_TIMEOUT,
      });

      test(
        "binds the container class as a durable object namespace",
        Effect.gen(function* () {
          const { url } = yield* stack;

          const body = yield* fetchReady(new URL("/binding", url), "kind");
          expect(JSON.parse(body)).toEqual({
            kind: "durable_object_namespace",
          });
        }).pipe(logLevel),
        { tags: [...(dev ? ["local"] : ["live"])], timeout },
      );

      test(
        "serves through the container-backed DO class",
        Effect.gen(function* () {
          const { url } = yield* stack;

          const hello = yield* fetchReady(new URL("/hello", url), "method");
          expect(hello).toContain("method");
        }).pipe(logLevel),
        { tags: [...(dev ? ["local"] : ["live"])], timeout },
      );
    });
  },
  {
    tags: [
      "provider:cloudflare",
      "provider:cloudflare:container",
      "provider:cloudflare:worker",
    ],
  },
);

describe.sequential(
  "container attachment recovery (live)",
  {
    tags: [
      "provider:cloudflare",
      "provider:cloudflare:container",
      "provider:cloudflare:worker",
      "live",
    ],
  },
  () => {
    const stage = process.env.ALCHEMY_TEST_STAGE ?? "test-live";
    const providers = Cloudflare.providers();
    const services = Layer.effectContext(
      AttachmentStack.pipe(Effect.map((stack) => stack.services)),
    ).pipe(Layer.provide(Layer.succeed(Stage, stage)));
    const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
      providers,
      state: Cloudflare.state(),
      stage,
      dev: false,
    });
    const stack = beforeAll(deploy(AttachmentStack), { timeout: 120_000 });
    afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(AttachmentStack), {
      timeout: 120_000,
    });
    const workerKey = {
      stack: attachmentStackName,
      stage,
      fqn: "AttachmentWorker",
    };
    const containerKey = {
      stack: attachmentStackName,
      stage,
      fqn: "AttachmentContainer",
    };

    test(
      "preserves the live attachment when the Worker projection and cached attachment are stale",
      Effect.gen(function* () {
        const first = yield* stack;
        const namespaceId = first.app.durableObjects?.namespaceId;
        const otherNamespaceId = first.otherApp.durableObjects?.namespaceId;
        assert(namespaceId);
        assert(otherNamespaceId);
        expect(otherNamespaceId).not.toBe(namespaceId);
        const detached = yield* attachmentStack(false).pipe(
          Effect.flatMap((stack) =>
            Plan.make(stack).pipe(Effect.provide(stack.services)),
          ),
        );
        expect(detached.resources.AttachmentContainer.action).toBe("replace");
        const state = yield* yield* State;
        const workerRow = yield* state.get(workerKey);
        const containerRow = yield* state.get(containerKey);
        assert(
          workerRow?.status === "created" || workerRow?.status === "updated",
        );
        assert(
          containerRow?.status === "created" ||
            containerRow?.status === "updated",
        );
        for (const durableObjects of [
          { namespaceId: otherNamespaceId },
          undefined,
        ]) {
          yield* Effect.gen(function* () {
            yield* state.set({
              ...workerKey,
              value: {
                ...workerRow,
                attr: { ...workerRow.attr, durableObjectNamespaces: {} },
              },
            });
            yield* state.set({
              ...containerKey,
              value: {
                ...containerRow,
                attr: {
                  ...containerRow.attr,
                  durableObjects,
                  hash: { ...containerRow.attr.hash, image: "stale" },
                },
              },
            });
            const plan = yield* AttachmentStack.pipe(
              Effect.flatMap((stack) =>
                Plan.make(stack).pipe(Effect.provide(stack.services)),
              ),
            );
            expect(plan.resources.AttachmentContainer.action).toBe("update");
            const next = yield* deploy(AttachmentStack);
            expect(next.worker.durableObjectNamespaces).toEqual(
              first.worker.durableObjectNamespaces,
            );
            expect(next.app.applicationId).toBe(first.app.applicationId);
            expect(next.app.durableObjects).toEqual({ namespaceId });
            const observed = yield* readAttachmentApplication(
              first.app.accountId,
              first.app.applicationId,
            );
            expect(observed.durableObjects).toEqual({ namespaceId });
            expect(observed.maxInstances).toBe(2);
            expect(next.otherApp.applicationId).toBe(
              first.otherApp.applicationId,
            );
            const other = yield* readAttachmentApplication(
              first.otherApp.accountId,
              first.otherApp.applicationId,
            );
            expect(other.durableObjects).toEqual({
              namespaceId: otherNamespaceId,
            });
            expect(other.maxInstances).toBe(2);
            const response = yield* fetchReady(
              new URL("/hello", next.url),
              "method",
              8,
            );
            expect(JSON.parse(response).method).toBe("GET");
          }).pipe(
            Effect.ensuring(
              Effect.gen(function* () {
                const current = yield* state.get(containerKey);
                if (
                  current &&
                  "attr" in current &&
                  current.attr?.applicationId &&
                  current.attr.applicationId !== first.app.applicationId
                ) {
                  yield* destroyStack({
                    stack: AttachmentStack,
                    stage,
                    dev: false,
                  });
                }
                yield* state
                  .set({ ...workerKey, value: workerRow })
                  .pipe(
                    Effect.ensuring(
                      state
                        .set({ ...containerKey, value: containerRow })
                        .pipe(Effect.orDie),
                    ),
                  );
              }).pipe(Effect.orDie),
            ),
          );
        }
      }).pipe(Effect.provide(services)),
      { timeout: 120_000, retry: 0 },
    );

    test(
      "recovers an interrupted create with an unresolved Worker attachment",
      Effect.gen(function* () {
        const first = yield* stack;
        const namespaceId = first.app.durableObjects?.namespaceId;
        assert(namespaceId);
        const state = yield* yield* State;
        const workerRow = yield* state.get(workerKey);
        const containerRow = yield* state.get(containerKey);
        assert(
          workerRow?.status === "created" || workerRow?.status === "updated",
        );
        assert(
          containerRow?.status === "created" ||
            containerRow?.status === "updated",
        );
        const interrupted = {
          ...containerRow,
          status: "creating" as const,
          attr: undefined,
        };
        yield* Effect.gen(function* () {
          yield* state.set({
            ...workerKey,
            value: {
              ...workerRow,
              attr: { ...workerRow.attr, durableObjectNamespaces: {} },
            },
          });
          yield* state.set({ ...containerKey, value: interrupted });
          const plan = yield* AttachmentStack.pipe(
            Effect.flatMap((stack) =>
              Plan.make(stack).pipe(Effect.provide(stack.services)),
            ),
          );
          expect(plan.resources.AttachmentContainer).toMatchObject({
            action: "create",
            state: {
              instanceId: containerRow.instanceId,
              attr: {
                applicationId: first.app.applicationId,
                applicationName: first.app.applicationName,
                durableObjects: { namespaceId },
              },
            },
          });
          expect(yield* state.get(containerKey)).toEqual(interrupted);
          const recovered = yield* deploy(AttachmentStack);
          expect(recovered.app.applicationId).toBe(first.app.applicationId);
          expect(recovered.app.durableObjects).toEqual({ namespaceId });
          expect(recovered.worker.durableObjectNamespaces).toEqual(
            first.worker.durableObjectNamespaces,
          );
          const committed = yield* state.get(containerKey);
          assert(
            committed?.status === "created" || committed?.status === "updated",
          );
          expect(committed).toMatchObject({
            instanceId: containerRow.instanceId,
            attr: {
              applicationId: first.app.applicationId,
              durableObjects: { namespaceId },
            },
          });
          const observed = yield* readAttachmentApplication(
            first.app.accountId,
            first.app.applicationId,
          );
          expect(observed.durableObjects).toEqual({ namespaceId });
          const other = yield* readAttachmentApplication(
            first.otherApp.accountId,
            first.otherApp.applicationId,
          );
          expect(other.durableObjects).toEqual(first.otherApp.durableObjects);
          expect(recovered.otherApp.applicationId).toBe(
            first.otherApp.applicationId,
          );
          const response = yield* fetchReady(
            new URL("/hello", recovered.url),
            "method",
            8,
          );
          expect(JSON.parse(response).method).toBe("GET");
        }).pipe(
          Effect.ensuring(
            Effect.gen(function* () {
              const current = yield* state.get(containerKey);
              if (
                current &&
                "attr" in current &&
                current.attr?.applicationId &&
                current.attr.applicationId !== first.app.applicationId
              ) {
                yield* destroyStack({
                  stack: AttachmentStack,
                  stage,
                  dev: false,
                });
              }
              yield* state
                .set({ ...workerKey, value: workerRow })
                .pipe(
                  Effect.ensuring(
                    state
                      .set({ ...containerKey, value: containerRow })
                      .pipe(Effect.orDie),
                  ),
                );
            }).pipe(Effect.orDie),
          ),
        );
      }).pipe(Effect.provide(services)),
      { timeout: 120_000, retry: 0 },
    );

    test(
      "recovers a missing application only with its recorded exact attachment",
      Effect.gen(function* () {
        const first = yield* stack;
        const namespaceId = first.app.durableObjects?.namespaceId;
        assert(namespaceId);
        const state = yield* yield* State;
        const workerRow = yield* state.get(workerKey);
        const containerRow = yield* state.get(containerKey);
        assert(
          workerRow?.status === "created" || workerRow?.status === "updated",
        );
        assert(
          containerRow?.status === "created" ||
            containerRow?.status === "updated",
        );
        const corruptProjection = (
          durableObjects: { namespaceId?: string } | undefined,
        ) =>
          Effect.gen(function* () {
            yield* state.set({
              ...workerKey,
              value: {
                ...workerRow,
                attr: { ...workerRow.attr, durableObjectNamespaces: {} },
              },
            });
            yield* state.set({
              ...containerKey,
              value: {
                ...containerRow,
                attr: {
                  ...containerRow.attr,
                  durableObjects,
                  hash: { ...containerRow.attr.hash, image: "stale" },
                },
              },
            });
          });
        yield* Effect.gen(function* () {
          yield* Containers.deleteContainerApplication({
            accountId: first.app.accountId,
            applicationId: first.app.applicationId,
          });
          const remaining = yield* Containers.listContainerApplications({
            accountId: first.app.accountId,
          }).pipe(
            Effect.repeat({
              schedule: Schedule.spaced("1 second"),
              until: (apps) =>
                apps.every((app) => app.id !== first.app.applicationId),
              times: 8,
            }),
          );
          expect(
            remaining.some((app) => app.id === first.app.applicationId),
          ).toBe(false);
          // Reconcile observes by id first; list deletion can propagate earlier.
          const deleted = yield* Containers.getContainerApplication({
            accountId: first.app.accountId,
            applicationId: first.app.applicationId,
          }).pipe(
            Effect.tap((app) =>
              Effect.logInfo(
                `Deleted container application ${app.id} is absent from list but still readable by id`,
              ),
            ),
            Effect.catchTag("ContainerApplicationNotFound", () =>
              Effect.succeed(undefined),
            ),
            Effect.repeat({
              schedule: Schedule.spaced("1 second"),
              until: (app) => app === undefined,
              times: 8,
            }),
          );
          expect(deleted).toBeUndefined();
          for (const missing of [undefined, {}]) {
            yield* corruptProjection(missing);
            const failed = yield* deploy(AttachmentStack).pipe(Effect.exit);
            assert(Exit.isFailure(failed));
            expect(Cause.pretty(failed.cause)).toContain(
              "unresolved Durable Object namespace",
            );
            const applications = yield* Containers.listContainerApplications({
              accountId: first.app.accountId,
            });
            expect(
              applications.some(
                (app) => app.name === first.app.applicationName,
              ),
            ).toBe(false);
          }
          yield* corruptProjection({ namespaceId });
          const recovered = yield* deploy(AttachmentStack);
          expect(recovered.worker.durableObjectNamespaces).toEqual(
            first.worker.durableObjectNamespaces,
          );
          expect(recovered.app.applicationId).not.toBe(first.app.applicationId);
          expect(recovered.app.durableObjects).toEqual({ namespaceId });
          const observed = yield* readAttachmentApplication(
            first.app.accountId,
            recovered.app.applicationId,
          );
          expect(observed.durableObjects).toEqual({ namespaceId });
          expect(observed.maxInstances).toBe(2);
          const other = yield* readAttachmentApplication(
            first.otherApp.accountId,
            first.otherApp.applicationId,
          );
          expect(other.durableObjects).toEqual(first.otherApp.durableObjects);
          const response = yield* fetchReady(
            new URL("/hello", recovered.url),
            "method",
            8,
          );
          expect(JSON.parse(response).method).toBe("GET");
        }).pipe(
          Effect.ensuring(
            Effect.gen(function* () {
              // Delete the replacement while its current identity is still persisted.
              yield* destroyStack({
                stack: AttachmentStack,
                stage,
                dev: false,
              });
              yield* Effect.gen(function* () {
                const remaining = yield* Containers.listContainerApplications({
                  accountId: first.app.accountId,
                }).pipe(
                  Effect.repeat({
                    schedule: Schedule.spaced("1 second"),
                    until: (apps) =>
                      apps.every(
                        (app) =>
                          app.name !== first.app.applicationName &&
                          app.name !== first.otherApp.applicationName,
                      ),
                    times: 8,
                  }),
                );
                expect(
                  remaining.some(
                    (app) =>
                      app.name === first.app.applicationName ||
                      app.name === first.otherApp.applicationName,
                  ),
                ).toBe(false);
              }).pipe(
                Effect.ensuring(
                  state
                    .set({ ...workerKey, value: workerRow })
                    .pipe(
                      Effect.ensuring(
                        state
                          .set({ ...containerKey, value: containerRow })
                          .pipe(Effect.orDie),
                      ),
                      Effect.orDie,
                    ),
                ),
              );
            }).pipe(Effect.orDie),
          ),
        );
      }).pipe(Effect.provide(services)),
      { timeout: 120_000, retry: 0 },
    );
  },
);

const readAttachmentApplication = (accountId: string, applicationId: string) =>
  Containers.getContainerApplication({ accountId, applicationId }).pipe(
    Effect.retry({
      while: (error) => error._tag === "ContainerApplicationNotFound",
      schedule: Schedule.spaced("1 second"),
      times: 8,
    }),
  );

// Note on image choice for the non-Effect (`image`/`dockerfile`) variants:
// stock nginx images crash-loop inside Cloudflare's container sandbox because
// they symlink /var/log/nginx/{access,error}.log to /dev/stdout and
// /dev/stderr, and opening those device paths fails with ENXIO (errno 6) — the
// nginx master aborts with `[emerg] could not open error log file` before it
// ever binds the port, so getTcpPort() can never connect. The fixtures work
// around this two ways: the `external` Dockerfile (which we own) replaces the
// log symlinks with real files, and the `remote` variant uses a server image
// (`mendhak/http-https-echo`) that writes to its inherited stdout fd directly
// instead of opening the /dev/std* paths as files.

// Cap exponential backoff at 3s — keeps the fast path snappy but stops the
// geometric blow-up from dominating wall time when CF edge is slow.
const readinessSchedule = Schedule.min([
  Schedule.exponential("500 millis"),
  Schedule.spaced("3 seconds"),
]);
// ~3 min of 3s-spaced polls: a saturated full-suite run (many concurrent
// docker builds/pushes on this machine) can keep a dev container answering
// 500 well past the ~90s that 30 retries buys. Failing attempts return
// fast, so the extra budget only matters on the slow path.
const readinessRetries = 60;

// While a freshly pre-created worker propagates, Cloudflare's edge serves
// Alchemy's pre-create stub (200 with this body); any poll that sees it retries.
const DEPLOY_PLACEHOLDER = "Alchemy worker is being deployed...";

// Force `Connection: close` so each readiness attempt opens a fresh connection
// and can land on an edge that already has the new deploy (a pooled keep-alive
// socket stays pinned to one edge metal and can keep reading the stale body).
const freshConn = HttpClient.HttpClient.pipe(
  Effect.map(
    HttpClient.mapRequest(HttpClientRequest.setHeader("connection", "close")),
  ),
);

// Retry a freshly-deployed worker route until it answers 200 with a body that
// contains `expected` — rejecting both transient non-200s and the deploy stub.
// Each attempt is bounded so a worker that is hung waiting on its container
// surfaces as a retryable failure rather than blocking the whole test budget.
const fetchReady = (url: URL, expected: string, retries = readinessRetries) =>
  Effect.gen(function* () {
    const client = yield* freshConn;
    return yield* client.get(url).pipe(
      Effect.flatMap((r) =>
        r.text.pipe(
          Effect.flatMap((body) =>
            r.status !== 200
              ? Effect.fail(new Error(`Worker not ready: ${r.status} ${body}`))
              : body.includes(DEPLOY_PLACEHOLDER) || !body.includes(expected)
                ? Effect.fail(new Error(`not ready: got ${body}`))
                : Effect.succeed(body),
          ),
        ),
      ),
      Effect.timeout("10 seconds"),
      Effect.retry({ schedule: readinessSchedule, times: retries }),
    );
  });

// Seed a value into R2 via the worker's `/seed` route (DO native binding),
// retrying through cold-start until the producer accepts it (200).
const seed = (url: string, key: string, value: string) =>
  Effect.gen(function* () {
    const client = yield* freshConn;
    return yield* client
      .execute(
        HttpClientRequest.put(
          new URL(`/seed?key=${encodeURIComponent(key)}`, url),
        ).pipe(HttpClientRequest.bodyText(value)),
      )
      .pipe(
        Effect.flatMap((r) =>
          r.status === 200
            ? Effect.succeed(r)
            : Effect.fail(new Error(`seed not ready: ${r.status}`)),
        ),
        Effect.timeout("10 seconds"),
        Effect.retry({
          schedule: readinessSchedule,
          times: readinessRetries,
        }),
      );
  });