{
  "id": "osc-stack-data-pipeline-orchestration",
  "job": "Store data, orchestrate scheduled ETL, and query it at scale, without hand-wiring storage, scheduling and the analytics database together.",
  "aliases": [
    "data pipeline orchestration",
    "airflow etl on osc",
    "object storage plus data warehouse",
    "scheduled data workflow"
  ],
  "inputs": [
    {
      "protocol": "file/S3",
      "source": "raw data files landed in S3-compatible object storage, or rows written directly to the relational store"
    }
  ],
  "outputs": [
    {
      "protocol": "SQL",
      "consumer": "ClickHouse for high-volume analytical queries, or PostgreSQL for transactional/metadata reads",
      "scheduledBy": "Airflow DAGs on a cron or sensor trigger"
    }
  ],
  "recommendedPath": {
    "chain": [
      {
        "step": 1,
        "serviceId": "minio-minio",
        "role": "object storage / data lake landing zone",
        "requiredConfig": ["name"],
        "requiresCredential": false,
        "note": "RootUser and RootPassword are recommended, not required (verified live against the schema 2026-09-29, same correction #1074 made for MinIO's chain position: only name is required at create time)."
      },
      {
        "step": 2,
        "serviceId": "birme-osc-postgresql",
        "role": "relational store for pipeline metadata and transactional tables",
        "requiredConfig": ["name", "PostgresPassword"],
        "requiresCredential": true,
        "note": "PostgresUser and PostgresDb are optional and default to 'postgres' / the username."
      },
      {
        "step": 3,
        "serviceId": "apache-airflow",
        "role": "orchestrate scheduled DAGs against the storage and database instances above",
        "requiredConfig": ["name"],
        "requiresCredential": false,
        "note": "AdminPassword and DatabaseUrl are optional at create time. DatabaseUrl is where the Postgres instance's connection string would be supplied to move Airflow's own metadata off SQLite, but no service association links this automatically: it must be filled in by hand or by the deploying agent after the Postgres instance exists."
      }
    ],
    "alternativePath": {
      "serviceId": "clickhouse-clickhouse",
      "role": "swap in for step 2, or add alongside it, when the destination needs petabyte-scale columnar analytics rather than transactional rows",
      "requiredConfig": ["name"],
      "requiresCredential": false,
      "note": "Db, User and Password are all optional at create time; wiring Airflow's DAG output to it is application code, not a platform-level connection."
    },
    "workflowAutomationAlternative": {
      "serviceId": "automatisch-automatisch",
      "role": "no-code workflow automation across 90-plus apps, as an alternative to hand-written Airflow DAGs for simpler triggers",
      "requiredConfig": ["name", "RedisUrl", "PostgresUrl"],
      "requiresCredential": false,
      "note": "Both RedisUrl and PostgresUrl are REQUIRED and must point at already-deployed instances. valkey-io-valkey (requiredConfig: name only) is the catalog's Redis-protocol-compatible store for RedisUrl; whether Automatisch actually works against Valkey rather than a literal Redis instance has not been tested end to end and is UNVERIFIED, not carried from any source."
    },
    "tokensPerDay": null,
    "tokensPerDayMeasured": false,
    "measuredWallClockSeconds": null,
    "wallClockMeasured": false,
    "gapNote": "No author's own end-to-end run has been timed or costed for this stack. No number is invented here; this is a named gap, not a filled-in figure."
  },
  "configSurface": {
    "servicesInChain": 3,
    "totalRequiredFieldsAcrossChain": 4,
    "sensitiveFields": ["PostgresPassword"],
    "note": "None of the four services checked (minio-minio, birme-osc-postgresql, apache-airflow, clickhouse-clickhouse) carry a serviceAssociation to another; every cross-service connection string (DatabaseUrl, RedisUrl, PostgresUrl) is filled in by hand or by the deploying agent, not wired by the platform."
  },
  "setup": {
    "codeRequired": true,
    "knownDeployGotcha": "Deploy minio-minio and birme-osc-postgresql first; apache-airflow's DatabaseUrl and automatisch-automatisch's RedisUrl/PostgresUrl fields both require an already-running dependency's connection details, and none of it auto-populates. codeRequired is true because Airflow DAGs and any ClickHouse-destination wiring are hand-written Python/application code, not configuration."
  },
  "cost": {
    "unit": "tokens/day",
    "ceiling": null,
    "ceilingMeasured": false,
    "note": "Not costed as a named stack yet, per the P5 CEO decision that a stack states its required tier up front once that measurement exists. Named gap for the next author to close, not an invented figure."
  },
  "servicesVerifiedLive": {
    "date": "2026-09-29",
    "method": "osc_call_tool wrap (mcp__osc__osc_call_tool, name=get-service-schema) on minio-minio, birme-osc-postgresql, apache-airflow, clickhouse-clickhouse, automatisch-automatisch, valkey-io-valkey; direct mcp__osc__get-service-schema now errors under operator mode, per the same operator-mode move CLAUDE.md already records for the catalog service count.",
    "catalogTotal": 187,
    "catalogTotalSource": "carried from today's dispatch-plan catalog-drift-check (2026-09-29, exit 0, per-category sum verified), not independently re-derived in this session"
  },
  "gateCheck": {
    "gatedServiceId": "eyevinn-just-go-live",
    "usesGatedService": false,
    "note": "This stack does not name or link eyevinn-just-go-live; the P3 gate does not bind it."
  },
  "verifiedOn": "2026-09-29",
  "publishedAt": "https://www.osaas.io/use-cases/data-pipelines",
  "descriptorUrl": "https://www.osaas.io/stacks/data-pipeline-orchestration.json"
}