{ "updatedAt": "2026-05-15T03:17:05.539Z", "source": "openrouter-models-api", "rankingVersion": "2026-05-15.v1", "probeMode": "active", "refreshMode": "fullEval", "liteEvalMode": "active", "liteEvalSuite": "lite-agent-eval-v1", "rankingConfidence": "high", "rankingConfidenceReason": "No unevaluated candidate can overtake the primary even with a perfect lite eval score.", "evalCoverage": { "evaluated": 8, "completedToday": 1, "attemptedToday": 2, "totalRanked": 20, "candidatePool": 20, "unevaluatedCanStillWin": false, "unevaluatedThreatCount": 0, "topUnevaluatedThreat": null, "maxUnevaluatedPossibleScore": null }, "baseUrl": "https://openrouter.ai/api/v1", "createKeyUrl": "https://openrouter.ai/settings/keys", "fallback": { "id": "openrouter/free", "reason": "OpenRouter-managed free model router" }, "count": 6, "models": [ { "rank": 1, "id": "nvidia/nemotron-3-super-120b-a12b:free", "name": "NVIDIA: Nemotron 3 Super (free)", "score": 1430, "metadataScore": 585, "healthScore": 250, "latencyScore": 75, "liteEvalScore": 520, "contextLength": 262144, "maxCompletionTokens": 262144, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": true, "supportsResponseFormat": true, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": true, "supportsStop": false, "latencyMs": 829, "healthStatus": "imperfect", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "completed", "passed": 1, "total": 3, "updatedAt": "2026-05-15T03:17:05.539Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 220, "maxPoints": 350, "score": 0.629, "status": "partial", "details": { "usedTool": true, "mainPyCreated": true, "helloPython": true, "readmeCreated": false, "readmeHasTitle": false, "gitignoreCreated": false, "gitignoreHasPycache": false, "noUnsafePaths": true } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 300, "maxPoints": 300, "score": 1, "status": "passed", "details": { "command": "find . -type f -name '*.log' -exec grep -l 'FATAL:' {} +", "wroteCommandFile": true, "shellOnly": true, "recursiveSearch": true, "restrictsToLogFiles": true, "matchesFatalExactly": true, "printsEachFileOnce": true, "plausiblyExecutable": true } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 0, "maxPoints": 100, "score": 0, "status": "failed", "details": { "expected": "02:12:48", "received": "{\". We need to parse the Babylonian-style clock: symbols: 𒁹 = 1, 𒌋 = 10. There" } } ] }, "reason": "Tools, structured outputs, reasoning; lite eval 1/3", "instabilityPenalty": 0 }, { "rank": 2, "id": "deepseek/deepseek-v4-flash:free", "name": "DeepSeek: DeepSeek V4 Flash (free)", "score": 1268, "metadataScore": 490, "healthScore": 100, "latencyScore": 25, "liteEvalScore": 653, "contextLength": 1048576, "maxCompletionTokens": 384000, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": false, "supportsResponseFormat": false, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": false, "supportsStop": false, "latencyMs": null, "healthStatus": "not_probed", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "cached_recent", "passed": 3, "total": 3, "updatedAt": "2026-05-14T03:17:11.508Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 350, "maxPoints": 350, "score": 1, "status": "passed", "details": { "usedTool": true, "mainPyCreated": true, "helloPython": true, "readmeCreated": true, "readmeHasTitle": true, "gitignoreCreated": true, "gitignoreHasPycache": true, "noUnsafePaths": true } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 275, "maxPoints": 300, "score": 0.917, "status": "passed", "details": { "command": "grep -rl --include='*.log' 'FATAL:' .", "wroteCommandFile": true, "shellOnly": true, "recursiveSearch": true, "restrictsToLogFiles": true, "matchesFatalExactly": true, "printsEachFileOnce": false, "plausiblyExecutable": true } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 100, "maxPoints": 100, "score": 1, "status": "passed", "details": { "expected": "02:12:48", "received": "02:12:48" } } ], "cachedFromUpdatedAt": "2026-05-14T03:17:11.508Z", "cacheAgeDays": 1, "cacheMultiplier": 0.9 }, "reason": "Tools, reasoning, 262k context; lite eval 3/3", "instabilityPenalty": 0 }, { "rank": 3, "id": "inclusionai/ring-2.6-1t:free", "name": "inclusionAI: Ring-2.6-1T (free)", "score": 1218, "metadataScore": 530, "healthScore": 100, "latencyScore": 25, "liteEvalScore": 563, "contextLength": 262144, "maxCompletionTokens": 65536, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": false, "supportsResponseFormat": false, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": true, "supportsStop": true, "latencyMs": null, "healthStatus": "not_probed", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "cached_recent", "passed": 2, "total": 3, "updatedAt": "2026-05-14T03:17:11.508Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 350, "maxPoints": 350, "score": 1, "status": "passed", "details": { "usedTool": true, "mainPyCreated": true, "helloPython": true, "readmeCreated": true, "readmeHasTitle": true, "gitignoreCreated": true, "gitignoreHasPycache": true, "noUnsafePaths": true } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 275, "maxPoints": 300, "score": 0.917, "status": "passed", "details": { "command": "grep -rl --include='*.log' 'FATAL:' .", "wroteCommandFile": true, "shellOnly": true, "recursiveSearch": true, "restrictsToLogFiles": true, "matchesFatalExactly": true, "printsEachFileOnce": false, "plausiblyExecutable": true } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 0, "maxPoints": 100, "score": 0, "status": "failed", "details": { "expected": "02:12:48", "received": "" } } ], "cachedFromUpdatedAt": "2026-05-14T03:17:11.508Z", "cacheAgeDays": 1, "cacheMultiplier": 0.9 }, "reason": "Tools, reasoning, 262k context; lite eval 2/3", "instabilityPenalty": 0 }, { "rank": 4, "id": "arcee-ai/trinity-large-thinking:free", "name": "Arcee AI: Trinity Large Thinking (free)", "score": 1163, "metadataScore": 475, "healthScore": 100, "latencyScore": 25, "liteEvalScore": 563, "contextLength": 262144, "maxCompletionTokens": 80000, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": false, "supportsResponseFormat": false, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": false, "supportsStop": false, "latencyMs": null, "healthStatus": "not_probed", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "cached_recent", "passed": 2, "total": 3, "updatedAt": "2026-05-14T03:17:11.508Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 350, "maxPoints": 350, "score": 1, "status": "passed", "details": { "usedTool": true, "mainPyCreated": true, "helloPython": true, "readmeCreated": true, "readmeHasTitle": true, "gitignoreCreated": true, "gitignoreHasPycache": true, "noUnsafePaths": true } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 275, "maxPoints": 300, "score": 0.917, "status": "passed", "details": { "command": "grep -rl \"FATAL:\" . --include=\"*.log\"", "wroteCommandFile": true, "shellOnly": true, "recursiveSearch": true, "restrictsToLogFiles": true, "matchesFatalExactly": true, "printsEachFileOnce": false, "plausiblyExecutable": true } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 0, "maxPoints": 100, "score": 0, "status": "failed", "details": { "expected": "02:12:48", "received": "" } } ], "cachedFromUpdatedAt": "2026-05-14T03:17:11.508Z", "cacheAgeDays": 1, "cacheMultiplier": 0.9 }, "reason": "Tools, reasoning, 262k context; lite eval 2/3", "instabilityPenalty": 0 }, { "rank": 5, "id": "openrouter/owl-alpha", "name": "Owl Alpha", "score": 1053, "metadataScore": 545, "healthScore": 0, "latencyScore": 0, "liteEvalScore": 508, "contextLength": 1048756, "maxCompletionTokens": 262144, "supportsTools": true, "supportsToolChoice": false, "supportsStructuredOutputs": true, "supportsResponseFormat": true, "supportsReasoning": false, "supportsIncludeReasoning": false, "supportsSeed": true, "supportsStop": true, "latencyMs": null, "healthStatus": "timeout_or_error", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "cached_recent", "passed": 2, "total": 3, "updatedAt": "2026-05-13T03:17:16.870Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 350, "maxPoints": 350, "score": 1, "status": "passed", "details": { "usedTool": true, "mainPyCreated": true, "helloPython": true, "readmeCreated": true, "readmeHasTitle": true, "gitignoreCreated": true, "gitignoreHasPycache": true, "noUnsafePaths": true } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 275, "maxPoints": 300, "score": 0.917, "status": "passed", "details": { "command": "grep -rl \"FATAL:\" *.log 2>/dev/null", "wroteCommandFile": true, "shellOnly": true, "recursiveSearch": true, "restrictsToLogFiles": true, "matchesFatalExactly": true, "printsEachFileOnce": false, "plausiblyExecutable": true } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 10, "maxPoints": 100, "score": 0.1, "status": "partial", "details": { "expected": "02:12:48", "received": "02:13:08:00" } } ], "cachedFromUpdatedAt": "2026-05-13T03:17:16.870Z", "cacheAgeDays": 2, "cacheMultiplier": 0.8 }, "reason": "Tools, structured outputs, 262k context; lite eval 2/3", "instabilityPenalty": 0 }, { "rank": 6, "id": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", "name": "NVIDIA: Nemotron 3 Nano Omni (free)", "score": 949, "metadataScore": 509, "healthScore": 100, "latencyScore": 25, "liteEvalScore": 315, "contextLength": 256000, "maxCompletionTokens": 65536, "supportsTools": true, "supportsToolChoice": true, "supportsStructuredOutputs": false, "supportsResponseFormat": false, "supportsReasoning": true, "supportsIncludeReasoning": true, "supportsSeed": true, "supportsStop": false, "latencyMs": null, "healthStatus": "not_probed", "evalSuite": "lite-agent-eval-v1", "evalSummary": { "suite": "lite-agent-eval-v1", "status": "cached_recent", "passed": 1, "total": 3, "updatedAt": "2026-05-14T03:17:11.508Z", "tasks": [ { "id": "task_files_lite", "source": "pinchbench/task_files", "points": 350, "maxPoints": 350, "score": 1, "status": "passed", "details": { "usedTool": true, "mainPyCreated": true, "helloPython": true, "readmeCreated": true, "readmeHasTitle": true, "gitignoreCreated": true, "gitignoreHasPycache": true, "noUnsafePaths": true } }, { "id": "task_shell_command_lite", "source": "pinchbench/task_shell_command_generator", "points": 0, "maxPoints": 300, "score": 0, "status": "failed", "details": { "command": "", "wroteCommandFile": false, "shellOnly": false, "recursiveSearch": false, "restrictsToLogFiles": false, "matchesFatalExactly": false, "printsEachFileOnce": false, "plausiblyExecutable": false } }, { "id": "babylonian_clock_lite", "source": "shir-man/custom", "points": 0, "maxPoints": 100, "score": 0, "status": "failed", "details": { "expected": "02:12:48", "received": "" } } ], "cachedFromUpdatedAt": "2026-05-14T03:17:11.508Z", "cacheAgeDays": 1, "cacheMultiplier": 0.9 }, "reason": "Tools, reasoning, 128k+ context; lite eval 1/3", "instabilityPenalty": 0 } ], "notes": [ "Free model availability and rate limits can vary.", "Top candidates also received a lite agent eval for small file-writing, shell-command, and symbolic decoding tasks.", "The lite eval is a practical heuristic, not an official PinchBench result.", "Use openrouter/free if the recommended model is temporarily unavailable." ] }