import from monorepo

worker/orca-agent/tests/config.yaml

@@ -0,0 +1,26 @@
+# Test Configuration
+task_id: "62n2aAVVV42rtt53wxieotTdnKpTRjiChsHYdSxHDhAZ" # Task ID from config-task.yml
+middle_server_url: "http://localhost:3000"
+# base_port: 5000 # Base port for worker servers
+# max_rounds: 1 # Maximum number of test rounds
+
+# Paths
+# relative to the test directory
+data_dir: data/minimal # Directory containing test data
+# workers_config: workers.json # Worker configuration file
+
+# MongoDB Configuration
+mongodb:
+  database: summarizer
+  collections:
+    docs:
+      data_file: docs.json # Relative to data_dir
+      required_count: 1 # Minimum number of documents required
+    summaries:
+      data_file: summaries.json
+      required_count: 1
+    systemprompts:
+      data_file: prompts.json
+      required_count: 1
+    audits:
+      required_count: 0 # No data file, just needs to exist

worker/orca-agent/tests/data/minimal/docs.json

@@ -0,0 +1,10 @@
+[
+  {
+    "taskId": "62n2aAVVV42rtt53wxieotTdnKpTRjiChsHYdSxHDhAZ",
+    "content": "This is a test document that needs to be summarized. It contains multiple sentences and paragraphs to test the summarization functionality. The document discusses various topics and should be processed by the workers to generate a concise summary.",
+    "metadata": {
+      "source": "test",
+      "type": "text"
+    }
+  }
+]

worker/orca-agent/tests/data/minimal/prompts.json

@@ -0,0 +1,7 @@
+[
+  {
+    "taskId": "62n2aAVVV42rtt53wxieotTdnKpTRjiChsHYdSxHDhAZ",
+    "prompt": "Please provide a concise summary of the following document, focusing on the main points and key information.",
+    "type": "summarization"
+  }
+]

worker/orca-agent/tests/data/minimal/summaries.json

@@ -0,0 +1,10 @@
+[
+  {
+    "taskId": "62n2aAVVV42rtt53wxieotTdnKpTRjiChsHYdSxHDhAZ",
+    "summary": "Test document containing multiple sentences for summarization testing.",
+    "metadata": {
+      "source": "test",
+      "type": "summary"
+    }
+  }
+]

worker/orca-agent/tests/e2e.py

@@ -0,0 +1,67 @@
+"""End-to-end test for the summarizer task."""
+
+from pathlib import Path
+from prometheus_test import TestRunner
+import dotenv
+import argparse
+import uuid
+
+
+dotenv.load_dotenv()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Run summarizer test sequence")
+    parser.add_argument(
+        "--reset",
+        action="store_true",
+        help="Force reset of all databases before running tests",
+    )
+    return parser.parse_args()
+
+
+def add_uuids(db):
+    """Post-load callback to process MongoDB data after JSON import"""
+    # Process docs collection
+    docs = list(db.docs.find({"taskId": runner.config.task_id}))
+    for doc in docs:
+        if "uuid" not in doc:
+            doc["uuid"] = str(uuid.uuid4())
+        db.docs.replace_one({"_id": doc["_id"]}, doc)
+
+    # Process summaries collection
+    summaries = list(db.summaries.find({"taskId": runner.config.task_id}))
+    for summary in summaries:
+        if "uuid" not in summary:
+            summary["uuid"] = str(uuid.uuid4())
+        if "docUuid" not in summary and docs:
+            # Link to first doc for simplicity
+            summary["docUuid"] = docs[0]["uuid"]
+        db.summaries.replace_one({"_id": summary["_id"]}, summary)
+
+
+# Global reference to the test runner
+runner = None
+
+
+def main():
+    global runner
+    args = parse_args()
+
+    # Import steps here to avoid circular imports
+    from .steps import steps
+
+    # Create test runner with config from YAML
+    base_dir = Path(__file__).parent
+    runner = TestRunner(
+        steps=steps,
+        config_file=base_dir / "config.yaml",
+        config_overrides={"post_load_callback": add_uuids},
+    )
+
+    # Run test sequence
+    runner.run(force_reset=args.reset)
+
+
+if __name__ == "__main__":
+    main()

worker/orca-agent/tests/stages/worker_fetch.py

@@ -0,0 +1,72 @@
+"""Stage for executing worker tasks."""
+
+import requests
+from prometheus_test.utils import create_signature
+
+
+def prepare(runner, worker):
+    """Prepare data for worker task"""
+    # Create fetch-todo payload for stakingSignature and publicSignature
+    payload = {
+        "taskId": runner.config.task_id,
+        "roundNumber": runner.current_round,
+        "action": "fetch-todo",
+        "githubUsername": worker.env.get("GITHUB_USERNAME"),
+        "stakingKey": worker.staking_public_key,
+        "pubKey": worker.public_key,
+    }
+
+    return {
+        "taskId": runner.config.task_id,
+        "roundNumber": runner.current_round,
+        "stakingKey": worker.staking_public_key,
+        "pubKey": worker.public_key,
+        "stakingSignature": create_signature(worker.staking_signing_key, payload),
+        "publicSignature": create_signature(worker.public_signing_key, payload),
+    }
+
+
+def execute(runner, worker, data):
+    """Execute worker task step"""
+    url = f"{worker.url}/worker-task/{data['roundNumber']}"
+    response = requests.post(
+        url,
+        json={"signature": data["stakingSignature"], "stakingKey": data["stakingKey"]},
+    )
+    result = response.json()
+
+    # Handle 409 gracefully - no eligible todos is an expected case
+    if response.status_code == 409:
+        print(
+            f"✓ {result.get('message', 'No eligible todos')} for {worker.name} - continuing"
+        )
+        return {"success": True, "message": result.get("message")}
+
+    if result.get("success") and "pr_url" in result:
+        round_key = str(runner.current_round)
+        round_state = runner.state["rounds"].setdefault(round_key, {})
+
+        # Initialize pr_urls if not exists
+        if "pr_urls" not in round_state:
+            round_state["pr_urls"] = {}
+        round_state["pr_urls"][worker.name] = result["pr_url"]
+
+        # Initialize submission_data if not exists
+        if "submission_data" not in round_state:
+            round_state["submission_data"] = {}
+
+        # Store submission data
+        round_state["submission_data"][worker.name] = {
+            "githubUsername": worker.env.get("GITHUB_USERNAME"),
+            "nodeType": "worker",
+            "prUrl": result["pr_url"],
+            "repoName": result.get("repoName"),
+            "repoOwner": result.get("repoOwner"),
+            "roundNumber": runner.current_round,
+            "taskId": runner.config.task_id,
+            "uuid": result.get("uuid"),  # Should be provided by the worker
+            "stakingKey": worker.staking_public_key,
+            "pubKey": worker.public_key,
+        }
+
+    return result

worker/orca-agent/tests/steps.py

@@ -0,0 +1,42 @@
+"""Test step definitions."""
+
+from prometheus_test import TestStep
+from functools import partial
+from .stages import (
+    worker_task,
+    worker_submission,
+    worker_audit,
+    audit_results,
+)
+
+
+steps = [
+    TestStep(
+        name="worker_task",
+        description="Execute worker task",
+        prepare=worker_task.prepare,
+        execute=worker_task.execute,
+        worker="worker",
+    ),
+    TestStep(
+        name="worker_submission",
+        description="Submit worker task",
+        prepare=worker_submission.prepare,
+        execute=worker_submission.execute,
+        worker="worker1",
+    ),
+    TestStep(
+        name="worker_audit",
+        description="Worker2 audits Worker1",
+        prepare=partial(worker_audit.prepare, target_name="worker1"),
+        execute=worker_audit.execute,
+        worker="worker2",
+    ),
+    TestStep(
+        name="audit_results",
+        description="Update audit results",
+        prepare=partial(audit_results.prepare, role="worker"),
+        execute=audit_results.execute,
+        worker="worker1",
+    ),
+]

worker/orca-agent/tests/workers.json

@@ -0,0 +1,35 @@
+{
+  "leader": {
+    "env_vars": {
+      "GITHUB_TOKEN": "LEADER_GITHUB_TOKEN",
+      "GITHUB_USERNAME": "LEADER_GITHUB_USERNAME",
+      "ANTHROPIC_API_KEY": "LEADER_ANTHROPIC_API_KEY"
+    },
+    "keypairs": {
+      "staking": "LEADER_STAKING_KEYPAIR",
+      "public": "LEADER_PUBLIC_KEYPAIR"
+    }
+  },
+  "worker1": {
+    "env_vars": {
+      "GITHUB_TOKEN": "WORKER1_GITHUB_TOKEN",
+      "GITHUB_USERNAME": "WORKER1_GITHUB_USERNAME",
+      "ANTHROPIC_API_KEY": "WORKER1_ANTHROPIC_API_KEY"
+    },
+    "keypairs": {
+      "staking": "WORKER1_STAKING_KEYPAIR",
+      "public": "WORKER1_PUBLIC_KEYPAIR"
+    }
+  },
+  "worker2": {
+    "env_vars": {
+      "GITHUB_TOKEN": "WORKER2_GITHUB_TOKEN",
+      "GITHUB_USERNAME": "WORKER2_GITHUB_USERNAME",
+      "ANTHROPIC_API_KEY": "WORKER2_ANTHROPIC_API_KEY"
+    },
+    "keypairs": {
+      "staking": "WORKER2_STAKING_KEYPAIR",
+      "public": "WORKER2_PUBLIC_KEYPAIR"
+    }
+  }
+}