{ "meta": { "attribution": "Anchor Terminal (https://www.anchorterminal.com)", "docs": "https://www.anchorterminal.com/docs/", "generatedAt": "2026-10-14", "license": "CC-BY-4.0", "method": "https://www.anchorterminal.com/benchmark/", "methodology": "0.4", "openapi": "https://www.anchorterminal.com/openapi.json", "preview": false, "run": "2026-10-01", "runLabel": "October 2026 research run" }, "tool": { "slug": "robot-framework", "name": "Robot Framework", "vendor": "Robot Framework Foundation", "vendorUrl": "https://robotframework.org", "kind": "framework", "category": "desktop-automation", "summary": "Robot Framework is an open-source automation framework for test automation and robotic process automation, run from the `robot` command line or Python. Its keyword libraries reach browsers, desktop applications and mobile apps.", "url": "https://www.anchorterminal.com/tools/robot-framework", "markdownUrl": "https://www.anchorterminal.com/tools/robot-framework.md", "slimMarkdownUrl": "https://www.anchorterminal.com/tools/robot-framework.min.md", "jsonUrl": "https://www.anchorterminal.com/api/v1/tools/robot-framework.json", "repo": "https://github.com/robotframework/robotframework", "license": "Apache-2.0", "transports": [], "packages": [ { "registry": "pypi", "name": "robotframework" } ], "auth": "none", "authNotes": "No account or key. The framework runs on the owner's machine. Credentials for the systems a suite touches come from variables, variable files or environment variables the owner supplies, and each library handles its own logins.", "pricing": "free", "pricingNotes": "Free and open source under Apache-2.0, with nothing to buy and no hosted service from the foundation. Companies fund it through foundation membership (checked 2026-10-14).", "priceSummary": "Free ยท OSS", "where": "library", "x402": { "level": "no", "evidence": "No x402, MPP or L402 in the source, the User Guide or robotframework.org (checked 2026-10-14).", "endpoints": [] }, "toolCount": null, "popularity": { "githubStars": 11939, "npmWeekly": null, "pypiWeekly": 775658, "asOf": "2026-10-14" }, "docsUrl": "https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html", "llmsTxt": "https://robotframework.org/llms.txt", "capabilities": [ "desktop.scripting", "desktop.rpa", "mobile.control" ], "tags": [ "open-source", "local", "cli", "python", "free", "no-card", "testing", "rpa", "llms-txt", "json-output" ], "lastRelease": "2026-09-14", "graded": true, "anchor": { "graded": true, "score": 76.4, "grade": "BB", "agentReady": true, "rank": 35, "ranked": true, "rankOf": 1076, "categoryRank": 1, "methodology": "0.4", "run": "2026-10-01", "scores": { "ergonomics": 78, "maintenance": 90, "payments": 60, "reliability": 95, "schema": 86, "security": 50, "transparency": 76 }, "pending": [ "performance", "tasks" ], "breakdown": [ { "key": "reliability", "name": "Reliability", "weight": 16, "effectiveWeight": 20, "score": 95, "points": 19, "reason": "Scored as local software. Official PyPI package with Python 3.8 or later and classifiers up to 3.15 (20). Unit tests and acceptance tests on CPython and PyPy run in public GitHub Actions and passed on master on 6 October 2026 (25). 311 open issues and pull requests on a project started on GitHub in 2014. Of the 15 newest, 7 had at least one reply, and none reports a crash (20 of 25). Release notes carry a backwards incompatible changes section and a deprecated functionality section, and the project follows major.minor feature releases with bug-fix releases (15). Version 7.5, Development Status Production/Stable (15)." }, { "key": "performance", "name": "Performance", "weight": 10, "effectiveWeight": 0, "pending": true, "points": 0, "reason": "Pending. Latency is measured per call by our probes, which haven't run yet, so this run doesn't score it. Its weight is shared across the assessed categories until the first probe window closes." }, { "key": "schema", "name": "Schema \u0026 documentation", "weight": 13, "effectiveWeight": 16.25, "score": 86, "points": 13.98, "reason": "Graded as a framework. JSON schemas for results, executable suites and Libdoc library specs, and XSDs for XML output, in doc/schema. There is no tool schema for an agent because no official MCP server exists (20 of 25). llms.txt, llms-full.txt and a raw Markdown copy of each page on robotframework.org (10). The User Guide explains each option and when to use it (16 of 20). Libdoc specs type keyword arguments, and keyword argument type hints drive conversion (12 of 15). Many examples, and return codes are documented in a table (13 of 15). Versioned releases with full release notes for each (15)." }, { "key": "ergonomics", "name": "Agent ergonomics", "weight": 13, "effectiveWeight": 16.25, "score": 78, "points": 12.68, "reason": "The command line adds no tool definitions to an agent's context, and `--loglevel`, `--removekeywords` and `--maxerrorlines` size the output (20 of 25). `--test`, `--suite`, `--include`, `--exclude` and `--rerunfailed` select what runs (20). The exit code counts failures and distinguishes data errors (252) and internal errors (255), and messages name the failing keyword (16 of 20). `--dryrun` validates a suite without side effects and `--rerunfailed` repeats only failures. Whether a keyword is safe to retry depends on the library (12 of 20). Few required arguments and a Python API, with other languages reached through the remote library interface rather than official SDKs (10 of 15)." }, { "key": "security", "name": "Security \u0026 auth", "weight": 14, "effectiveWeight": 17.5, "score": 50, "points": 8.75, "reason": "Graded as a framework. It has no credential of its own. Secrets come from variables, files or the environment, and log.html records keyword arguments unless `--removekeywords` strips them (15 of 30). `--dryrun` checks a suite without running keywords. No sandbox or approval hook, and suites run with the owner's full rights (8 of 20). It returns no untrusted content itself, though libraries read pages and windows (10 of 15). output.xml or output.json and log.html record every keyword, its arguments and status (13 of 15). No SECURITY.md, no security.txt on robotframework.org and no repository advisories found (4 of 20)." }, { "key": "payments", "name": "Payments \u0026 pricing", "weight": 10, "effectiveWeight": 12.5, "score": 60, "points": 7.5, "reason": "Free, self-hosted, nothing to buy, so 20 + 20 + 20 under the self-hosted rule. No payment protocol (0)." }, { "key": "tasks", "name": "Task success", "weight": 10, "effectiveWeight": 0, "pending": true, "points": 0, "reason": "Pending. Task success needs the category task suites run through each tool, which haven't run yet, so this run doesn't score it. Its weight is shared across the assessed categories until then. A data provider's data-quality score is published on its listing now and becomes half of this category when it's scored." }, { "key": "maintenance", "name": "Maintenance \u0026 community", "weight": 7, "effectiveWeight": 8.75, "score": 90, "points": 7.88, "reason": "Version 7.5 on 14 September 2026, 30 days before this check (30). 7.5b1, 7.5rc1 and 7.5 since 17 July 2026 (20). 162 commits since 14 July, and recent issues and pull requests get replies from maintainers within days, with some newer ones unanswered (20 of 25). Current official package on PyPI with no MCP server, so not in the MCP registry (10 of 15). Dependabot updates and passing CI on master (10)." }, { "key": "transparency", "name": "Transparency \u0026 trust", "weight": 7, "effectiveWeight": 8.75, "score": 76, "points": 6.65, "note": "editorial 78, provenance 74", "reason": "Apache-2.0 (30). The framework runs locally and collects nothing found in the source. The privacy policy covers only the websites, with Google Analytics and 50 months' retention stated (18 of 30). Deprecated functionality is listed in each release's notes with the version that removes them, such as the Telnet library in 7.5 (16 of 20). No telemetry code found in a search of the source, and the docs do not say so either way (14 of 20)." } ], "assessment": { "date": "2026-10-14", "basis": "public evidence", "confidence": "medium", "notes": { "ergonomics": "The command line adds no tool definitions to an agent's context, and `--loglevel`, `--removekeywords` and `--maxerrorlines` size the output (20 of 25). `--test`, `--suite`, `--include`, `--exclude` and `--rerunfailed` select what runs (20). The exit code counts failures and distinguishes data errors (252) and internal errors (255), and messages name the failing keyword (16 of 20). `--dryrun` validates a suite without side effects and `--rerunfailed` repeats only failures. Whether a keyword is safe to retry depends on the library (12 of 20). Few required arguments and a Python API, with other languages reached through the remote library interface rather than official SDKs (10 of 15).", "maintenance": "Version 7.5 on 14 September 2026, 30 days before this check (30). 7.5b1, 7.5rc1 and 7.5 since 17 July 2026 (20). 162 commits since 14 July, and recent issues and pull requests get replies from maintainers within days, with some newer ones unanswered (20 of 25). Current official package on PyPI with no MCP server, so not in the MCP registry (10 of 15). Dependabot updates and passing CI on master (10).", "payments": "Free, self-hosted, nothing to buy, so 20 + 20 + 20 under the self-hosted rule. No payment protocol (0).", "reliability": "Scored as local software. Official PyPI package with Python 3.8 or later and classifiers up to 3.15 (20). Unit tests and acceptance tests on CPython and PyPy run in public GitHub Actions and passed on master on 6 October 2026 (25). 311 open issues and pull requests on a project started on GitHub in 2014. Of the 15 newest, 7 had at least one reply, and none reports a crash (20 of 25). Release notes carry a backwards incompatible changes section and a deprecated functionality section, and the project follows major.minor feature releases with bug-fix releases (15). Version 7.5, Development Status Production/Stable (15).", "schema": "Graded as a framework. JSON schemas for results, executable suites and Libdoc library specs, and XSDs for XML output, in doc/schema. There is no tool schema for an agent because no official MCP server exists (20 of 25). llms.txt, llms-full.txt and a raw Markdown copy of each page on robotframework.org (10). The User Guide explains each option and when to use it (16 of 20). Libdoc specs type keyword arguments, and keyword argument type hints drive conversion (12 of 15). Many examples, and return codes are documented in a table (13 of 15). Versioned releases with full release notes for each (15).", "security": "Graded as a framework. It has no credential of its own. Secrets come from variables, files or the environment, and log.html records keyword arguments unless `--removekeywords` strips them (15 of 30). `--dryrun` checks a suite without running keywords. No sandbox or approval hook, and suites run with the owner's full rights (8 of 20). It returns no untrusted content itself, though libraries read pages and windows (10 of 15). output.xml or output.json and log.html record every keyword, its arguments and status (13 of 15). No SECURITY.md, no security.txt on robotframework.org and no repository advisories found (4 of 20).", "transparency": "Apache-2.0 (30). The framework runs locally and collects nothing found in the source. The privacy policy covers only the websites, with Google Analytics and 50 months' retention stated (18 of 30). Deprecated functionality is listed in each release's notes with the version that removes them, such as the Telnet library in 7.5 (16 of 20). No telemetry code found in a search of the source, and the docs do not say so either way (14 of 20)." }, "sources": [ { "what": "repository, source and doc/schema at commit of 6 October 2026", "url": "https://github.com/robotframework/robotframework", "seen": "2026-10-14" }, { "what": "Robot Framework 7.5 release notes", "url": "https://github.com/robotframework/robotframework/blob/master/doc/releasenotes/rf-7.5.rst", "seen": "2026-10-14" }, { "what": "tags (7.5b1, 7.5rc1, 7.5)", "url": "https://github.com/robotframework/robotframework/tags", "seen": "2026-10-14" }, { "what": "Actions runs on master", "url": "https://api.github.com/repos/robotframework/robotframework/actions/runs?branch=master", "seen": "2026-10-14" }, { "what": "stars and open issues", "url": "https://api.github.com/repos/robotframework/robotframework", "seen": "2026-10-14" }, { "what": "newest open issues and pull requests", "url": "https://api.github.com/repos/robotframework/robotframework/issues?state=open\u0026sort=created\u0026direction=desc", "seen": "2026-10-14" }, { "what": "PyPI package metadata", "url": "https://pypi.org/pypi/robotframework/json", "seen": "2026-10-14" }, { "what": "PyPI downloads", "url": "https://pypistats.org/api/packages/robotframework/recent", "seen": "2026-10-14" }, { "what": "home page", "url": "https://robotframework.org/", "seen": "2026-10-14" }, { "what": "llms.txt", "url": "https://robotframework.org/llms.txt", "seen": "2026-10-14" }, { "what": "privacy policy", "url": "https://robotframework.org/raw/privacy-policy.md", "seen": "2026-10-14" }, { "what": "foundation page", "url": "https://robotframework.org/raw/foundation.md", "seen": "2026-10-14" }, { "what": "User Guide, return codes", "url": "https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html", "seen": "2026-10-14" } ], "openQuestions": [ "unchecked: whether any core library stores or redacts secrets in logs by default beyond `--removekeywords`", "The home page showed 7.4.2 on 14 October 2026, a month after 7.5." ] }, "negative": 0, "verdict": "A stable 7.5 release, passing unit and acceptance tests, JSON schemas for results and library docs, and an llms.txt make it a dependable base for agent-written automation. Desktop and mobile control come from separate community libraries, there is no official MCP server, and no security policy or security.txt was found.", "bestFor": "Agents that write or run keyword-driven test and RPA suites across web, desktop and mobile through libraries.", "strengths": [ "Version 7.5 of 14 September 2026, marked Production/Stable on PyPI, with Python 3.8 to 3.15 listed", "Unit and acceptance test workflows on CPython and PyPy passed on master on 6 October 2026", "JSON schemas for run results, executable suites and Libdoc library specs are published in doc/schema", "robotframework.org serves llms.txt, llms-full.txt and a raw Markdown copy of each page", "Release notes have separate sections for backwards incompatible changes and deprecated functionality" ], "weaknesses": [ "Desktop and mobile control depend on separate libraries, such as AppiumLibrary and AutoItLibrary, maintained outside the core project", "No official MCP server. MCP servers for Robot Framework found on PyPI and Glama are community projects", "No SECURITY.md, no security.txt on robotframework.org and no published repository advisories", "robotframework.org still showed version 7.4.2 on 14 October 2026, a month after 7.5 shipped" ], "agentNotes": [ "Install with `pip install robotframework` and check with `robot --version`. Add the library for the target app, such as AppiumLibrary, separately", "Read the exit code. It is the number of failed tests or tasks up to 250, and 252, 253 or 255 mean bad input, a user stop or an internal error", "Use `--dryrun` to validate a suite without running keywords, and `--rerunfailed output.xml` to repeat only failures", "Convert results with `rebot --output output.json output.xml` for JSON that matches doc/schema/result.json", "Keep secrets out of keyword arguments, because log.html records arguments unless keywords are removed with `--removekeywords`" ], "metrics": { "kind": "local", "measured": false }, "reviewCount": 0, "avgRating": 0, "history": [ { "basis": "public evidence", "confidence": "medium", "grade": "BB", "methodology": "0.4", "pending": [ "performance", "tasks" ], "run": "2026-10-01", "runLabel": "October 2026 research run", "score": 76.4 } ], "editorialScores": { "ergonomics": 78, "maintenance": 90, "payments": 60, "reliability": 95, "schema": 86, "security": 50, "transparency": 78 }, "provenanceScore": 74 }, "connect": { "install": "pip install robotframework" }, "letme": { "capability": "https://letme.dev/desktop.scripting", "tool": "https://letme.dev/robot-framework" }, "notable": [ "Version 7.5 released on 14 September 2026, after 7.5b1 on 17 July and 7.5rc1 on 4 September (https://github.com/robotframework/robotframework/blob/master/doc/releasenotes/rf-7.5.rst)", "Maintained by Robot Framework ry, a Finnish non-profit foundation funded by member companies, which says it pays the maintainers (https://robotframework.org/raw/foundation.md)", "11,939 GitHub stars and 311 open issues and pull requests on 14 October 2026 (https://api.github.com/repos/robotframework/robotframework)", "About 776,000 PyPI downloads in the week to 14 October 2026 (https://pypistats.org/api/packages/robotframework/recent)", "The home page lists AppiumLibrary for Android and iOS and AutoItLibrary for Windows GUIs among external libraries (https://robotframework.org/)", "No official MCP server. RobotMCP (`rf-mcp`) and others are community projects (https://pypi.org/project/rf-mcp/)" ], "area": "developer", "details": [ { "label": "Interfaces", "value": "`robot` and `rebot` command-line tools, a Python API (`robot.run`, `robot.api`) and a remote library interface for keywords in other languages" }, { "label": "Machine-readable formats", "value": "JSON schemas for results (`result.json`), executable suites (`running_suite.json`) and Libdoc specs (`libdoc.json`), plus XSDs for XML output (https://github.com/robotframework/robotframework/tree/master/doc/schema)" }, { "label": "Return codes", "value": "0 when everything passes, 1 to 249 for that many failures, 250 for 250 or more, 251 help or version, 252 invalid data or options, 253 stopped by the user, 255 unexpected internal error (User Guide, Executing test cases)" }, { "label": "Desktop and mobile", "value": "Not in the core. External libraries such as AppiumLibrary (Android, iOS), AutoItLibrary (Windows) and RPA Framework's RPA.Windows and RPA.Desktop" }, { "label": "Runtimes", "value": "Python 3.8 or later, with classifiers up to 3.15, and PyPy" }, { "label": "Telemetry", "value": "None found in the source. Not stated either way in the documentation" }, { "label": "Licence", "value": "Apache-2.0. Copyright 2008 to 2015 Nokia Networks, 2016 onwards Robot Framework Foundation" } ], "provenance": { "legalEntity": "Robot Framework ry (Robot Framework Foundation), business ID 2754775-1, Helsinki", "domain": "robotframework.org", "domainRegistered": "2007-12-31", "domainNote": "RDAP for robotframework.org gives a registration date of 2007-12-31. robotframework.org/.well-known/security.txt returns 404.", "endpointOnVendorDomain": null, "terms": "", "privacy": "https://robotframework.org/privacy-policy/", "statusPage": "", "changelog": "https://github.com/robotframework/robotframework/tree/master/doc/releasenotes", "securityTxt": "none", "checked": "2026-10-14", "notes": [ "No terms of service govern the software beyond the Apache-2.0 licence, so `terms` is left out.", "The privacy policy covers the robotframework.org and RoboCon websites, not the framework, which runs on the owner's machine. It names Google Analytics and keeps analytics data for 50 months.", "COPYRIGHT.txt reads Copyright 2008 to 2015 Nokia Networks and 2016 onwards Robot Framework Foundation." ], "score": 74, "checks": [ { "check": "Legal entity named", "value": "Robot Framework ry (Robot Framework Foundation), business ID 2754775-1, Helsinki", "points": 20, "max": 20, "state": "ok" }, { "check": "Domain age", "value": "robotframework.org, registered 2007-12-31 (18 years)", "points": 15, "max": 15, "state": "ok" }, { "check": "Endpoint on the vendor's domain", "value": "no hosted endpoint", "points": 0, "max": 0, "state": "na" }, { "check": "Terms of service", "value": "nothing hosted, so the Apache-2.0 licence stands in", "points": 10, "max": 10, "state": "ok" }, { "check": "Privacy policy", "value": "read, states 5 of the 8 things a reader expects", "points": 7.8, "max": 10, "state": "part" }, { "check": "Status page", "value": "not found", "points": 0, "max": 10, "state": "no" }, { "check": "Changelog", "value": "published", "points": 10, "max": 10, "state": "ok" }, { "check": "security.txt", "value": "not found", "points": 0, "max": 10, "state": "no" } ], "policies": [ { "kind": "terms", "url": "", "state": "none-found", "points": 0, "max": 10 }, { "kind": "privacy", "url": "https://robotframework.org/privacy-policy/", "state": "read", "readAt": "2026-10-14", "words": 2208, "points": 7.8, "max": 10, "expected": [ { "key": "privacy.date", "label": "Gives the date it was last updated", "found": false }, { "key": "privacy.collected", "label": "Says what personal data is collected", "found": true, "quote": "We collect two types of information concerning our Users: (i) User Data;" }, { "key": "privacy.retention", "label": "Says how long data is kept", "found": true, "quote": "We will store Analytics Data relating to the Service 50 months.", "says": "Names a period of 50 months" }, { "key": "privacy.processors", "label": "Says who else receives the data", "found": true, "quote": "However, we have service providers in several geographical locations." }, { "key": "privacy.sale", "label": "Says whether personal data is sold or shared for advertising", "found": false }, { "key": "privacy.rights", "label": "Says what rights people have over their data", "found": true, "quote": "You have the right to withdraw this consent at all times." }, { "key": "privacy.contact", "label": "Gives a privacy contact", "found": false }, { "key": "privacy.transfers", "label": "Says where data is transferred or stored", "found": true, "quote": "We provide adequate protection for the transfers of personal data to countries outside of the European Economic Area through a series of agreements with our service providers based on the Standard Contractual Clauses or through other appropriate safeguards, such as the Privacy Shield Framework.", "says": "Relies on standard contractual clauses" } ] } ] }, "pageJsonUrl": "https://www.anchorterminal.com/tools/robot-framework.json" } }