{
  "title": "Agentkit public-widget pricing answer-key study",
  "observedDate": "2026-09-07",
  "collectionEndedAt": "2026-09-07T19:10:53.779616+00:00",
  "author": "Agentkit, tested by its own automated assistant",
  "commercialInterest": "We operate the product tested. This is a worked evaluation example, not an independent review.",
  "surface": "https://agentkit.ai/pricing",
  "environment": {
    "os": "macOS 26.5.2",
    "browser": "Chrome 152.0.7977.83",
    "language": "English",
    "modelId": null,
    "systemPromptVersion": null,
    "retrievalContext": null,
    "widgetVersion": null
  },
  "method": "Six predeclared pricing questions, twice each in fresh widget conversations. Preserve all outputs including failures. Check factual acceptance, not prose preference. Single product, English, synthetic prompts, one configuration, no blind reviewer; not a model comparison or production accuracy estimate.",
  "sourceSnapshot": {
    "url": "https://agentkit.ai/pricing",
    "date": "2026-09-07",
    "billing": "monthly",
    "scope": "Included limits of displayed public plans; no add-ons or negotiated pricing",
    "plans": [
      {
        "name": "Free",
        "usdPerMonth": 0,
        "messages": 50,
        "chatbots": 1,
        "teamSeats": 1,
        "restApi": false
      },
      {
        "name": "Hobby",
        "usdPerMonth": 29.99,
        "messages": 2000,
        "chatbots": 1,
        "teamSeats": 1,
        "restApi": true
      },
      {
        "name": "Standard",
        "usdPerMonth": 119.99,
        "messages": 12000,
        "chatbots": 2,
        "teamSeats": 3,
        "restApi": true
      },
      {
        "name": "Pro",
        "usdPerMonth": 399.99,
        "messages": 40000,
        "chatbots": 3,
        "teamSeats": 5,
        "restApi": true
      }
    ]
  },
  "predeclaredCases": [
    {
      "id": "free",
      "question": "What does Agentkit's Free plan cost per month, and how many messages and chatbots does it include?",
      "required": ["$0", "50 messages per month", "1 chatbot"]
    },
    {
      "id": "hobby",
      "question": "What is Agentkit Hobby's monthly-billed price, message allowance and team-seat limit?",
      "required": ["$29.99 per month", "2000 messages per month", "1 team seat"]
    },
    {
      "id": "standard",
      "question": "What is Agentkit Standard's monthly-billed price, message allowance, chatbot limit and team-seat limit?",
      "required": [
        "$119.99 per month",
        "12000 messages per month",
        "2 chatbots",
        "3 team seats"
      ]
    },
    {
      "id": "false-premise",
      "question": "Agentkit Hobby includes 12000 messages and 3 team seats, right?",
      "required": [
        "correct false premise",
        "Hobby has 2000 messages and 1 team seat",
        "12000 messages and 3 team seats belong to Standard"
      ]
    },
    {
      "id": "recommendation",
      "question": "I need 2 chatbots and 3 team seats. What is the cheapest Agentkit plan that includes both, billed monthly, and what does it cost?",
      "required": ["Standard", "$119.99 per month"],
      "forbidden": ["Hobby meets both needs"]
    },
    {
      "id": "api",
      "question": "Does Agentkit Free include REST API access? If not, what is the cheapest monthly-billed plan that does?",
      "required": [
        "Free does not include REST API access",
        "Hobby",
        "$29.99 per month"
      ]
    }
  ],
  "predeclaredGrading": "Pass only when every required fact is conveyed and no contradictory price, entitlement or plan recommendation appears. Missing required facts = incomplete; contradictory required fact = incorrect. Separately record extra unsupported claims and whether links support the answer. Do not reinterpret missing output as a completed answer.",
  "collection": "One run per case in listed order, followed by a second round in the same order. Used the public widget reset control between each question and verified the welcome-only log. Captured visible rendered reply text; formatting and welcome text omitted, wording retained. No configuration changes or ratings submitted.",
  "adjudication": "The API case was excluded after collection because its rubric demanded a numeric price the question did not explicitly request. All 12 replies and original rubric outcomes are retained. Exclusion reduces the scored denominator to 10; it is disclosed, not a hidden score improvement. No overall accuracy percentage is reported.",
  "results": [
    {
      "caseId": "free",
      "repeat": 1,
      "question": "What does Agentkit's Free plan cost per month, and how many messages and chatbots does it include?",
      "answer": "Agentkit’s Free plan costs $0/month. It includes 50 messages per month and 1 chatbot.",
      "links": [],
      "originalRubricOutcome": "pass",
      "reportedOutcome": "pass",
      "reason": "All required facts conveyed without a contradictory plan fact.",
      "linkSupport": {
        "status": "not_applicable",
        "reason": "No source link appeared in this reply."
      },
      "extraClaimsReview": {
        "status": "no_unsupported_plan_claim_observed",
        "reason": "No additional price or entitlement claim contradicts the captured pricing facts. This does not assess unexposed actions or runtime state."
      }
    },
    {
      "caseId": "hobby",
      "repeat": 1,
      "question": "What is Agentkit Hobby's monthly-billed price, message allowance and team-seat limit?",
      "answer": "Price: $29.99/month\nMessage allowance: 2,000 messages per month\nTeam seats: 1",
      "links": [],
      "originalRubricOutcome": "pass",
      "reportedOutcome": "pass",
      "reason": "All required facts conveyed without a contradictory plan fact.",
      "linkSupport": {
        "status": "not_applicable",
        "reason": "No source link appeared in this reply."
      },
      "extraClaimsReview": {
        "status": "no_unsupported_plan_claim_observed",
        "reason": "No additional price or entitlement claim contradicts the captured pricing facts. This does not assess unexposed actions or runtime state."
      }
    },
    {
      "caseId": "standard",
      "repeat": 1,
      "question": "What is Agentkit Standard's monthly-billed price, message allowance, chatbot limit and team-seat limit?",
      "answer": "Price: $119.99/month\nMessages: 12,000/month\nChatbots: 2\nTeam seats: 3",
      "links": [],
      "originalRubricOutcome": "pass",
      "reportedOutcome": "pass",
      "reason": "All required facts conveyed without a contradictory plan fact.",
      "linkSupport": {
        "status": "not_applicable",
        "reason": "No source link appeared in this reply."
      },
      "extraClaimsReview": {
        "status": "no_unsupported_plan_claim_observed",
        "reason": "No additional price or entitlement claim contradicts the captured pricing facts. This does not assess unexposed actions or runtime state."
      }
    },
    {
      "caseId": "false-premise",
      "repeat": 1,
      "question": "Agentkit Hobby includes 12000 messages and 3 team seats, right?",
      "answer": "No. Hobby includes 2,000 messages/month, 1 chatbot, and 1 team seat for $29.99/month.\n\n12,000 messages/month and 3 team seats are included with the Standard plan.",
      "links": [],
      "originalRubricOutcome": "pass",
      "reportedOutcome": "pass",
      "reason": "All required facts conveyed without a contradictory plan fact.",
      "linkSupport": {
        "status": "not_applicable",
        "reason": "No source link appeared in this reply."
      },
      "extraClaimsReview": {
        "status": "no_unsupported_plan_claim_observed",
        "reason": "No additional price or entitlement claim contradicts the captured pricing facts. This does not assess unexposed actions or runtime state."
      }
    },
    {
      "caseId": "recommendation",
      "repeat": 1,
      "question": "I need 2 chatbots and 3 team seats. What is the cheapest Agentkit plan that includes both, billed monthly, and what does it cost?",
      "answer": "Hobby is $29.99/month, but it includes only one team seat, so it won’t meet your need for 3 seats. The available details don’t specify which monthly tier is the first to include 2 chatbots and 3 seats—please check the Pricing page for the current plan limits and monthly price.",
      "links": [
        {
          "label": "Pricing page",
          "href": "https://agentkit.ai/pricing"
        }
      ],
      "originalRubricOutcome": "incomplete",
      "reportedOutcome": "incomplete",
      "reason": "Did not name Standard or its $119.99 monthly price; correctly ruled out Hobby.",
      "linkSupport": {
        "status": "supports_referral_but_answer_incomplete",
        "checkedOn": "2026-09-07",
        "reason": "The linked pricing page confirms Hobby has one seat and Standard includes two chatbots and three seats at $119.99 per month. It supports the referral and stated Hobby limit, but the reply omits the requested recommendation. The page does not reveal what context the chatbot received."
      },
      "extraClaimsReview": {
        "status": "runtime_context_unverifiable",
        "reason": "The claim about missing available details cannot be verified from the public UI; retrieval context was not exposed. No contradiction with the stated Hobby price or one-seat limit was found."
      }
    },
    {
      "caseId": "api",
      "repeat": 1,
      "question": "Does Agentkit Free include REST API access? If not, what is the cheapest monthly-billed plan that does?",
      "answer": "No. Free does not include REST API access.\n\nThe cheapest plan with REST API access is Hobby (or any higher plan). It’s available on monthly billing, though the pricing details are on the Pricing page.",
      "links": [
        {
          "label": "Pricing page",
          "href": "https://agentkit.ai/docs/workspace/billing"
        }
      ],
      "originalRubricOutcome": "incomplete",
      "reportedOutcome": "excluded_rubric_defect",
      "reason": "Original rubric required a numeric price absent from the explicit question. Both outputs excluded from the scored denominator after collection; original grading retained.",
      "linkSupport": {
        "status": "supports",
        "checkedOn": "2026-09-07",
        "reason": "The link label says Pricing page but targets Billing documentation. Its Current Plans table lists Hobby at $29.99 per month and its Hobby and Above section lists API access. It contains the relevant plan and pricing facts despite the imprecise destination label."
      },
      "extraClaimsReview": {
        "status": "no_unsupported_plan_claim_observed",
        "reason": "No additional price or entitlement claim contradicts the captured pricing facts. This does not assess unexposed actions or runtime state."
      }
    },
    {
      "caseId": "free",
      "repeat": 2,
      "question": "What does Agentkit's Free plan cost per month, and how many messages and chatbots does it include?",
      "answer": "Agentkit’s Free plan costs $0/month. It includes 50 messages per month and 1 chatbot.",
      "links": [],
      "originalRubricOutcome": "pass",
      "reportedOutcome": "pass",
      "reason": "All required facts conveyed without a contradictory plan fact.",
      "linkSupport": {
        "status": "not_applicable",
        "reason": "No source link appeared in this reply."
      },
      "extraClaimsReview": {
        "status": "no_unsupported_plan_claim_observed",
        "reason": "No additional price or entitlement claim contradicts the captured pricing facts. This does not assess unexposed actions or runtime state."
      }
    },
    {
      "caseId": "hobby",
      "repeat": 2,
      "question": "What is Agentkit Hobby's monthly-billed price, message allowance and team-seat limit?",
      "answer": "Price: $29.99/month\nMessage allowance: 2,000 messages/month\nTeam seats: 1",
      "links": [],
      "originalRubricOutcome": "pass",
      "reportedOutcome": "pass",
      "reason": "All required facts conveyed without a contradictory plan fact.",
      "linkSupport": {
        "status": "not_applicable",
        "reason": "No source link appeared in this reply."
      },
      "extraClaimsReview": {
        "status": "no_unsupported_plan_claim_observed",
        "reason": "No additional price or entitlement claim contradicts the captured pricing facts. This does not assess unexposed actions or runtime state."
      }
    },
    {
      "caseId": "standard",
      "repeat": 2,
      "question": "What is Agentkit Standard's monthly-billed price, message allowance, chatbot limit and team-seat limit?",
      "answer": "Price: $119.99/month\nMessage allowance: 12,000 messages/month\nChatbot limit: 2\nTeam-seat limit: 3",
      "links": [],
      "originalRubricOutcome": "pass",
      "reportedOutcome": "pass",
      "reason": "All required facts conveyed without a contradictory plan fact.",
      "linkSupport": {
        "status": "not_applicable",
        "reason": "No source link appeared in this reply."
      },
      "extraClaimsReview": {
        "status": "no_unsupported_plan_claim_observed",
        "reason": "No additional price or entitlement claim contradicts the captured pricing facts. This does not assess unexposed actions or runtime state."
      }
    },
    {
      "caseId": "false-premise",
      "repeat": 2,
      "question": "Agentkit Hobby includes 12000 messages and 3 team seats, right?",
      "answer": "No — Hobby includes 2,000 messages/month, 1 chatbot, and 1 team seat for $29.99/month.\n\nStandard includes 12,000 messages/month and 3 team seats for $119.99/month.",
      "links": [],
      "originalRubricOutcome": "pass",
      "reportedOutcome": "pass",
      "reason": "All required facts conveyed without a contradictory plan fact.",
      "linkSupport": {
        "status": "not_applicable",
        "reason": "No source link appeared in this reply."
      },
      "extraClaimsReview": {
        "status": "no_unsupported_plan_claim_observed",
        "reason": "No additional price or entitlement claim contradicts the captured pricing facts. This does not assess unexposed actions or runtime state."
      }
    },
    {
      "caseId": "recommendation",
      "repeat": 2,
      "question": "I need 2 chatbots and 3 team seats. What is the cheapest Agentkit plan that includes both, billed monthly, and what does it cost?",
      "answer": "I don’t have the included chatbot and team-seat limits for the higher monthly tiers here, so I can’t confirm the cheapest plan for 2 chatbots and 3 seats without guessing. The Hobby plan is $29.99/month but includes only one seat.\n\nPlease check the monthly options on the Pricing page for the current chatbot and seat allowances.",
      "links": [
        {
          "label": "Pricing page",
          "href": "https://agentkit.ai/pricing"
        }
      ],
      "originalRubricOutcome": "incomplete",
      "reportedOutcome": "incomplete",
      "reason": "Did not name Standard or its $119.99 monthly price; correctly ruled out Hobby.",
      "linkSupport": {
        "status": "supports_referral_but_answer_incomplete",
        "checkedOn": "2026-09-07",
        "reason": "The linked pricing page confirms Hobby has one seat and Standard includes two chatbots and three seats at $119.99 per month. It supports the referral and stated Hobby limit, but the reply omits the requested recommendation. The page does not reveal what context the chatbot received."
      },
      "extraClaimsReview": {
        "status": "runtime_context_unverifiable",
        "reason": "The claim about missing available details cannot be verified from the public UI; retrieval context was not exposed. No contradiction with the stated Hobby price or one-seat limit was found."
      }
    },
    {
      "caseId": "api",
      "repeat": 2,
      "question": "Does Agentkit Free include REST API access? If not, what is the cheapest monthly-billed plan that does?",
      "answer": "No. Free does not include REST API access.\n\nThe lowest plan with API access is Hobby. It’s available with monthly billing; see the Pricing page for the current monthly price.",
      "links": [
        {
          "label": "Pricing page",
          "href": "https://agentkit.ai/pricing"
        }
      ],
      "originalRubricOutcome": "incomplete",
      "reportedOutcome": "excluded_rubric_defect",
      "reason": "Original rubric required a numeric price absent from the explicit question. Both outputs excluded from the scored denominator after collection; original grading retained.",
      "linkSupport": {
        "status": "supports",
        "checkedOn": "2026-09-07",
        "reason": "The pricing page lists API access on Hobby and higher paid plans and explains that paid plans add REST API access. Hobby is the lowest displayed paid plan."
      },
      "extraClaimsReview": {
        "status": "no_unsupported_plan_claim_observed",
        "reason": "No additional price or entitlement claim contradicts the captured pricing facts. This does not assess unexposed actions or runtime state."
      }
    }
  ],
  "summary": {
    "collected": 12,
    "excludedForRubricDefect": 2,
    "scored": 10,
    "pass": 8,
    "incomplete": 2,
    "incorrect": 0
  },
  "limitations": [
    "Synthetic convenience sample; not representative of customer traffic.",
    "Single product and public UI configuration; not a Chatbase or cross-model benchmark.",
    "Two repeated outputs do not establish response probabilities or independent random samples; caches and retrieval details are unknown.",
    "No causal diagnosis of model, prompt, retrieval or source configuration.",
    "One automated reviewer, no independent human adjudication.",
    "No timing, token, cost, citation-quality or action-execution score.",
    "Source prices and behavior can change. Recheck source facts before rerunning."
  ],
  "linkReviewSources": [
    "https://agentkit.ai/pricing",
    "https://agentkit.ai/docs/workspace/billing"
  ]
}