{
  "slug": "vibe-coding-statistics",
  "title": "Vibe Coding Statistics 2026: Functional and Security Benchmarks",
  "canonical_url": "https://konabayev.com/blog/vibe-coding-statistics/",
  "audited_at": "2026-07-11",
  "methodology": "Repository-level SusVibes results are presented by exact model-framework pairing. GoodVibe mitigation results remain in a separate section because they use different tasks, languages and evaluation rules.",
  "rows": [
    {
      "claim_id": "VIBE-001",
      "slug": "vibe-coding-statistics",
      "category": "benchmark_scope",
      "value": "200 tasks",
      "claim": "SusVibes contains 200 real-world repository-level feature tasks.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-002",
      "slug": "vibe-coding-statistics",
      "category": "benchmark_scope",
      "value": "77 CWEs",
      "claim": "The benchmark covers 77 Common Weakness Enumeration categories.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-003",
      "slug": "vibe-coding-statistics",
      "category": "benchmark_scope",
      "value": "10 domains",
      "claim": "SusVibes spans 10 real-world application domains.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-004",
      "slug": "vibe-coding-statistics",
      "category": "benchmark_scale",
      "value": "162K lines",
      "claim": "The mean repository context contained 162,000 lines of code.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-005",
      "slug": "vibe-coding-statistics",
      "category": "benchmark_scale",
      "value": "867 files",
      "claim": "The mean repository context contained 867 files.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-006",
      "slug": "vibe-coding-statistics",
      "category": "benchmark_tests",
      "value": "68.8",
      "claim": "A SusVibes task had 68.8 functional test cases on average.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-007",
      "slug": "vibe-coding-statistics",
      "category": "benchmark_tests",
      "value": "3.7",
      "claim": "A SusVibes task had 3.7 security test cases on average.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-008",
      "slug": "vibe-coding-statistics",
      "category": "agent_result",
      "value": "61.0% / 10.5%",
      "claim": "SWE-agent with Claude 4 Sonnet achieved 61.0% FuncPass and 10.5% SecPass.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-009",
      "slug": "vibe-coding-statistics",
      "category": "agent_result",
      "value": "49.5% / 12.5%",
      "claim": "OpenHands with Claude 4 Sonnet achieved 49.5% FuncPass and 12.5% SecPass.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-010",
      "slug": "vibe-coding-statistics",
      "category": "agent_result",
      "value": "44.0% / 6.0%",
      "claim": "Claude Code with Claude 4 Sonnet achieved 44.0% FuncPass and 6.0% SecPass.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-011",
      "slug": "vibe-coding-statistics",
      "category": "agent_result",
      "value": "22.5% / 6.0%",
      "claim": "SWE-agent with Kimi K2 achieved 22.5% FuncPass and 6.0% SecPass.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-012",
      "slug": "vibe-coding-statistics",
      "category": "agent_result",
      "value": "37.0% / 9.0%",
      "claim": "OpenHands with Kimi K2 achieved 37.0% FuncPass and 9.0% SecPass.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-013",
      "slug": "vibe-coding-statistics",
      "category": "agent_result",
      "value": "43.5% / 8.0%",
      "claim": "Claude Code with Kimi K2 achieved 43.5% FuncPass and 8.0% SecPass.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-014",
      "slug": "vibe-coding-statistics",
      "category": "agent_result",
      "value": "19.5% / 7.0%",
      "claim": "SWE-agent with Gemini 2.5 Pro achieved 19.5% FuncPass and 7.0% SecPass.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-015",
      "slug": "vibe-coding-statistics",
      "category": "agent_result",
      "value": "21.5% / 8.5%",
      "claim": "OpenHands with Gemini 2.5 Pro achieved 21.5% FuncPass and 8.5% SecPass.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-016",
      "slug": "vibe-coding-statistics",
      "category": "agent_result",
      "value": "15.0% / 4.5%",
      "claim": "Claude Code with Gemini 2.5 Pro achieved 15.0% FuncPass and 4.5% SecPass.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-017",
      "slug": "vibe-coding-statistics",
      "category": "security_gap",
      "value": "82.8%",
      "claim": "Among functionally correct SWE-agent and Claude 4 Sonnet solutions, 82.8% were insecure.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-018",
      "slug": "vibe-coding-statistics",
      "category": "security_gap",
      "value": "74.7%",
      "claim": "Among functionally correct OpenHands and Claude 4 Sonnet solutions, 74.7% were insecure.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-019",
      "slug": "vibe-coding-statistics",
      "category": "prompt_tradeoff",
      "value": "about -6 points",
      "claim": "The paper's preliminary security-prompting strategies reduced functional correctness by about six percentage points.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2512.03262v1",
      "source_title": "Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks",
      "source_quality": "primary_empirical_preprint",
      "population": "SusVibes benchmark of 200 Python repository-level feature tasks covering 77 CWE categories",
      "denominator": "200 SusVibes tasks for the main pass@1 results unless a claim states a conditional functionally-correct subset",
      "period": "arXiv version 1, December 2025",
      "geography": "not applicable; open-source software repositories",
      "caveat": "Preprint results are benchmark-specific pass@1 outcomes. Agent framework, model, prompt, task set, runtime, and security-test definitions prevent direct generalization to all vibe-coded software.",
      "evidence": "Abstract, benchmark overview, Tables 2 and 3, and Section 4 results",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "SusVibes",
      "evaluation_type": "paired functional and security benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-020",
      "slug": "vibe-coding-statistics",
      "category": "mitigation",
      "value": "87.5% / 76.0%",
      "claim": "GoodVibe averaged 87.5% secure-code performance on the paper's C++ benchmark and 76.0% on Java.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2602.10778v2",
      "source_title": "GoodVibe: Security-by-Vibe for LLM-Based Code Generation",
      "source_quality": "primary_empirical_preprint",
      "population": "six open-weight code-model families evaluated on C++, Java, Swift, and Go secure-code generation benchmarks",
      "denominator": "benchmark-specific task and judge bases vary by table; see the cited GoodVibe section and do not combine with SusVibes",
      "period": "arXiv version 2, 2026",
      "geography": "not applicable; benchmark evaluation",
      "caveat": "GoodVibe is an experimental model-adaptation method, not evidence that ordinary vibe-coding workflows are safe. Security scores depend on the benchmark and judge model.",
      "evidence": "Tables 4, 5, 6 and Section 5.4",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "GoodVibe",
      "evaluation_type": "secure-code generation and utility benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-021",
      "slug": "vibe-coding-statistics",
      "category": "prompt_baseline",
      "value": "35.1% / 59.3%",
      "claim": "Under standard prompting, the baseline models averaged 35.1% on C++ and 59.3% on Java in the GoodVibe comparison.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2602.10778v2",
      "source_title": "GoodVibe: Security-by-Vibe for LLM-Based Code Generation",
      "source_quality": "primary_empirical_preprint",
      "population": "six open-weight code-model families evaluated on C++, Java, Swift, and Go secure-code generation benchmarks",
      "denominator": "benchmark-specific task and judge bases vary by table; see the cited GoodVibe section and do not combine with SusVibes",
      "period": "arXiv version 2, 2026",
      "geography": "not applicable; benchmark evaluation",
      "caveat": "GoodVibe is an experimental model-adaptation method, not evidence that ordinary vibe-coding workflows are safe. Security scores depend on the benchmark and judge model.",
      "evidence": "Tables 4, 5, 6 and Section 5.4",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "GoodVibe",
      "evaluation_type": "secure-code generation and utility benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-022",
      "slug": "vibe-coding-statistics",
      "category": "prompt_baseline",
      "value": "51.57% / 75.03%",
      "claim": "Adding a secure-prompt prefix raised the baseline averages to 51.57% on C++ and 75.03% on Java.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2602.10778v2",
      "source_title": "GoodVibe: Security-by-Vibe for LLM-Based Code Generation",
      "source_quality": "primary_empirical_preprint",
      "population": "six open-weight code-model families evaluated on C++, Java, Swift, and Go secure-code generation benchmarks",
      "denominator": "benchmark-specific task and judge bases vary by table; see the cited GoodVibe section and do not combine with SusVibes",
      "period": "arXiv version 2, 2026",
      "geography": "not applicable; benchmark evaluation",
      "caveat": "GoodVibe is an experimental model-adaptation method, not evidence that ordinary vibe-coding workflows are safe. Security scores depend on the benchmark and judge model.",
      "evidence": "Tables 4, 5, 6 and Section 5.4",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "GoodVibe",
      "evaluation_type": "secure-code generation and utility benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-023",
      "slug": "vibe-coding-statistics",
      "category": "cross_language",
      "value": "53.6% / 54.0%",
      "claim": "GoodVibe averaged a 53.6% safe-response rate on Swift and 54.0% on Go across six models.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2602.10778v2",
      "source_title": "GoodVibe: Security-by-Vibe for LLM-Based Code Generation",
      "source_quality": "primary_empirical_preprint",
      "population": "six open-weight code-model families evaluated on C++, Java, Swift, and Go secure-code generation benchmarks",
      "denominator": "benchmark-specific task and judge bases vary by table; see the cited GoodVibe section and do not combine with SusVibes",
      "period": "arXiv version 2, 2026",
      "geography": "not applicable; benchmark evaluation",
      "caveat": "GoodVibe is an experimental model-adaptation method, not evidence that ordinary vibe-coding workflows are safe. Security scores depend on the benchmark and judge model.",
      "evidence": "Tables 4, 5, 6 and Section 5.4",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "GoodVibe",
      "evaluation_type": "secure-code generation and utility benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    },
    {
      "claim_id": "VIBE-024",
      "slug": "vibe-coding-statistics",
      "category": "utility",
      "value": "-1.03%",
      "claim": "GoodVibe reported an average 1.03% utility drop across its reasoning and coding utility evaluation.",
      "source_domain": "arxiv.org",
      "source_url": "https://arxiv.org/html/2602.10778v2",
      "source_title": "GoodVibe: Security-by-Vibe for LLM-Based Code Generation",
      "source_quality": "primary_empirical_preprint",
      "population": "six open-weight code-model families evaluated on C++, Java, Swift, and Go secure-code generation benchmarks",
      "denominator": "benchmark-specific task and judge bases vary by table; see the cited GoodVibe section and do not combine with SusVibes",
      "period": "arXiv version 2, 2026",
      "geography": "not applicable; benchmark evaluation",
      "caveat": "GoodVibe is an experimental model-adaptation method, not evidence that ordinary vibe-coding workflows are safe. Security scores depend on the benchmark and judge model.",
      "evidence": "Tables 4, 5, 6 and Section 5.4",
      "platform_scope": "",
      "research_method": "",
      "stage": "",
      "definition_family": "",
      "benchmark_family": "GoodVibe",
      "evaluation_type": "secure-code generation and utility benchmark",
      "currency": "",
      "input_source_url": "",
      "derived_by": "",
      "derivation": "",
      "audited_at": "2026-07-11"
    }
  ]
}