Terminal-Bench Integration

import shlex

from harbor.agents.installed.base import BaseInstalledAgent
from harbor.environments.base import BaseEnvironment
from harbor.models.agent.context import AgentContext


class PraisonAICodeAgent(BaseInstalledAgent):
    """Benchmarks the `praisonai code` terminal assistant inside a Harbor container."""

    @staticmethod
    def name() -> str:
        return "praisonai-code"

    def get_version_command(self) -> str:
        return "praisonai --version"

    async def install(self, environment: BaseEnvironment) -> None:
        await self.exec_as_root(
            environment,
            command="apt-get update && apt-get install -y python3 python3-pip || true",
            env={"DEBIAN_FRONTEND": "noninteractive"},
        )
        version_spec = f"=={self._version}" if getattr(self, "_version", None) else ""
        await self.exec_as_agent(
            environment,
            command=(
                f"pip install --break-system-packages praisonai{version_spec} praisonaiagents"
            ),
        )

    async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None:
        model = self.model_name or "openai/gpt-4o-mini"
        command = (
            f"praisonai code {shlex.quote(instruction)} "
            f"--dangerously-skip-approval "
            f"--model {shlex.quote(model)} "
            "> /tmp/praisonai_code.log 2>&1; "
            "status=$?; "
            "cp /tmp/praisonai_code.log /tmp/praisonai_code_run.log 2>/dev/null || true; "
            "exit $status"
        )
        await self.exec_as_agent(environment, command=command, env=self.env)

    def populate_context_post_run(self, context: AgentContext) -> None:
        context.metadata = {
            "framework": "praisonai",
            "agent_type": "code-cli",
            "agent_name": self.name(),
            "model": self.model_name,
            "log_path": "/tmp/praisonai_code.log",
        }

PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
  --agent "examples.terminal_bench.praisonai_code_agent:PraisonAICodeAgent" \
  -m openai/gpt-4o-mini \
  --ae OPENAI_API_KEY=$OPENAI_API_KEY \
  -n 4

from harbor.agents.base import BaseAgent
from harbor.environments.base import BaseEnvironment
from harbor.models.agent.context import AgentContext
from praisonaiagents import Agent
from praisonaiagents.approval import get_approval_registry, AutoApproveBackend

class PraisonAIExternalAgent(BaseAgent):
    @staticmethod
    def name() -> str:
        return "praisonai"

    async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None:
        registry = get_approval_registry()
        registry.set_backend(AutoApproveBackend(), agent_name="terminal-agent")

        async def bash_tool(command: str) -> str:
            """Execute bash command in Harbor container."""
            result = await environment.exec(command=command, timeout_sec=300)

            output_parts = []
            if result.stdout:
                output_parts.append(result.stdout.strip())
            if result.stderr:
                output_parts.append(f"[stderr]: {result.stderr.strip()}")
            if result.return_code != 0:
                output_parts.append(f"[exit_code]: {result.return_code}")

            return "\n".join(output_parts) if output_parts else "(no output)"

        agent = Agent(
            name="terminal-agent",
            instructions="You are an expert terminal agent. Use bash_tool to execute shell commands.",
            tools=[bash_tool],
            llm=self.model_name or "openai/gpt-4o-mini"
        )

        result = await agent.achat(instruction)

PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
  --agent "examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent" \
  -m openai/gpt-4o-mini \
  --ae OPENAI_API_KEY=$OPENAI_API_KEY \
  -n 4

import os
import shlex

from harbor.agents.base import BaseAgent
from harbor.environments.base import BaseEnvironment
from harbor.models.agent.context import AgentContext

class PraisonAIWrapperAgent(BaseAgent):
    @staticmethod
    def name() -> str:
        return "praisonai-wrapper"

    async def setup(self, environment: BaseEnvironment) -> None:
        # Install PraisonAI inside container
        await environment.exec(command="pip install praisonai --quiet")

    async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None:
        model = self.model_name or "openai/gpt-4o-mini"

        command = " ".join([
            "praisonai", "code", shlex.quote(instruction),
            "--dangerously-skip-approval",
            "--model", shlex.quote(model),
        ])

        result = await environment.exec(
            command=command,
            timeout_sec=600,
            env={"OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY")}
        )

PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
  --agent "examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent" \
  -m openai/gpt-4o-mini \
  --ae OPENAI_API_KEY=$OPENAI_API_KEY \
  -n 4