llms-txt/index.html
github-actions[bot] 0ac1627140 deploy: 329488a27b
2026-08-26 22:59:03 +00:00

908 lines
No EOL
51 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
<meta charset="utf-8">
<meta name="generator" content="quarto-1.10.18">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
<meta name="author" content="Jeremy Howard">
<meta name="dcterms.date" content="2024-09-03">
<meta name="description" content="A proposal to standardise on using an /llms.txt file to provide information to help agents use a website.">
<title>The /llms.txt file, v2 – llms-txt</title>
<style>
/* Default styles provided by pandoc.
** See https://pandoc.org/MANUAL.html#variables-for-html for config info.
*/
span.smallcaps{font-variant: small-caps;}
div.columns{display: flex; gap: 1.5em;}
div.column{flex: auto;}
@media screen {
div.columns{gap: min(4vw, 1.5em);}
div.column{overflow-x: auto;}
}
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
ul.task-list{list-style: none;}
ul.task-list li input[type="checkbox"] {
width: 0.8em;
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
vertical-align: middle;
}
/* CSS for syntax highlighting */
html { -webkit-text-size-adjust: 100%; }
pre > code.sourceCode { white-space: pre; position: relative; }
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
pre > code.sourceCode > span:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode > span { color: inherit; text-decoration: inherit; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
pre > code.sourceCode { white-space: pre-wrap; }
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
}
pre.numberSource code
{ counter-reset: source-line 0; }
pre.numberSource code > span
{ position: relative; left: -4em; counter-increment: source-line; }
pre.numberSource code > span > a:first-child::before
{ content: counter(source-line);
position: relative; left: -1em; text-align: right; vertical-align: baseline;
border: none; display: inline-block;
-webkit-touch-callout: none; -webkit-user-select: none;
-khtml-user-select: none; -moz-user-select: none;
-ms-user-select: none; user-select: none;
padding: 0 4px; width: 4em;
}
pre.numberSource { margin-left: 3em; padding-left: 4px; }
div.sourceCode
{ }
@media screen {
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
}
</style>
<script src="site_libs/quarto-nav/quarto-nav.js"></script>
<script src="site_libs/quarto-nav/headroom.min.js"></script>
<script src="site_libs/clipboard/clipboard.min.js"></script>
<script src="site_libs/quarto-search/autocomplete.umd.js"></script>
<script src="site_libs/quarto-search/fuse.min.js"></script>
<script src="site_libs/quarto-search/quarto-search.js"></script>
<meta name="quarto:offset" content="./">
<link href="./favicon.ico" rel="icon">
<script src="site_libs/quarto-html/quarto.js" type="module"></script>
<script src="site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
<script src="site_libs/quarto-html/popper.min.js"></script>
<script src="site_libs/quarto-html/tippy.umd.min.js"></script>
<script src="site_libs/quarto-html/anchor.min.js"></script>
<link href="site_libs/quarto-html/tippy.css" rel="stylesheet">
<link href="site_libs/quarto-html/quarto-syntax-highlighting-64a204ab8560d761af7679ceaba22944.css" rel="stylesheet" id="quarto-text-highlighting-styles">
<script src="site_libs/bootstrap/bootstrap.min.js"></script>
<link href="site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
<link href="site_libs/bootstrap/bootstrap-947a84243c33a66514163c09ef13f91d.min.css" rel="stylesheet" append-hash="true" id="quarto-bootstrap" data-mode="light">
<script src="site_libs/quarto-contrib/glightbox/glightbox.min.js"></script>
<link href="site_libs/quarto-contrib/glightbox/glightbox.min.css" rel="stylesheet">
<link href="site_libs/quarto-contrib/glightbox/lightbox.css" rel="stylesheet">
<script id="quarto-search-options" type="application/json">{
"location": "navbar",
"copy-button": false,
"collapse-after": 3,
"panel-placement": "end",
"type": "overlay",
"limit": 50,
"keyboard-shortcut": [
"f",
"/",
"s"
],
"show-item-context": false,
"language": {
"search-no-results-text": "No results",
"search-matching-documents-text": "matching documents",
"search-copy-link-title": "Copy link to search",
"search-hide-matches-text": "Hide additional matches",
"search-more-match-text": "more match in this document",
"search-more-matches-text": "more matches in this document",
"search-clear-button-title": "Clear",
"search-text-placeholder": "",
"search-detached-cancel-button-title": "Cancel",
"search-submit-button-title": "Submit",
"search-label": "Search"
}
}</script>
<link rel="stylesheet" href="styles.css">
<meta property="og:title" content="The /llms.txt file, v2 – llms-txt">
<meta property="og:description" content="A proposal to standardise on using an /llms.txt file to provide information to help agents use a website.">
<meta property="og:image" content="https://llmstxt.org//sample.png">
<meta property="og:site_name" content="llms-txt">
<meta property="og:image:height" content="718">
<meta property="og:image:width" content="1108">
<meta name="twitter:title" content="The /llms.txt file, v2 – llms-txt">
<meta name="twitter:description" content="A proposal to standardise on using an /llms.txt file to provide information to help agents use a website.">
<meta name="twitter:image" content="https://llmstxt.org//sample.png">
<meta name="twitter:creator" content="@jeremyphoward">
<meta name="twitter:site" content="@answerdotai">
<meta name="twitter:image-height" content="718">
<meta name="twitter:image-width" content="1108">
<meta name="twitter:card" content="summary_large_image">
</head>
<body class="nav-sidebar floating nav-fixed quarto-light">
<div id="quarto-search-results"></div>
<header id="quarto-header" class="headroom fixed-top">
<nav class="navbar navbar-expand-lg " data-bs-theme="dark">
<div class="navbar-container container-fluid">
<div class="navbar-brand-container mx-auto">
<a class="navbar-brand" href="./index.html">
<span class="navbar-title">llms-txt</span>
</a>
</div>
<div id="quarto-search" class="" title="Search"></div>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse" id="navbarCollapse">
<ul class="navbar-nav navbar-nav-scroll ms-auto">
<li class="nav-item compact">
<a class="nav-link" href="https://github.com/answerdotai/llms-txt"> <i class="bi bi-github" role="img">
</i>
<span class="menu-text"></span></a>
</li>
<li class="nav-item compact">
<a class="nav-link" href="https://x.com/answerdotai"> <i class="bi bi-twitter" role="img" aria-label="Fast.ai Twitter">
</i>
<span class="menu-text"></span></a>
</li>
</ul>
</div> <!-- /navcollapse -->
<div class="quarto-navbar-tools">
</div>
</div> <!-- /container-fluid -->
</nav>
<nav class="quarto-secondary-nav">
<div class="container-fluid d-flex">
<button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
<i class="bi bi-layout-text-sidebar-reverse"></i>
</button>
<nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="./index.html">The /llms.txt file, v2</a></li></ol></nav>
<a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
</a>
</div>
</nav>
</header>
<!-- content -->
<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
<!-- sidebar -->
<nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation floating overflow-auto">
<div class="sidebar-menu-container">
<ul class="list-unstyled mt-1">
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./index.html" class="sidebar-item-text sidebar-link active">
<span class="menu-text">The /llms.txt file, v2</span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./changes.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">Changes</span></a>
</div>
</li>
<li class="sidebar-item sidebar-item-section">
<div class="sidebar-item-container">
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true">
<span class="menu-text">Code</span></a>
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true" aria-label="Toggle section">
<i class="bi bi-chevron-right ms-2"></i>
</a>
</div>
<ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./intro.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">Python module &amp; CLI</span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./core.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">Python source</span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./llmstxt-js.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">JavaScript</span></a>
</div>
</li>
</ul>
</li>
<li class="sidebar-item sidebar-item-section">
<div class="sidebar-item-container">
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true">
<span class="menu-text">Editors and IDEs</span></a>
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true" aria-label="Toggle section">
<i class="bi bi-chevron-right ms-2"></i>
</a>
</div>
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./ed.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><code>ed</code>, the standard text editor</span></a>
</div>
</li>
</ul>
</li>
<li class="sidebar-item sidebar-item-section">
<div class="sidebar-item-container">
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" role="navigation" aria-expanded="true">
<span class="menu-text">Tutorials</span></a>
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" role="navigation" aria-expanded="true" aria-label="Toggle section">
<i class="bi bi-chevron-right ms-2"></i>
</a>
</div>
<ul id="quarto-sidebar-section-3" class="collapse list-unstyled sidebar-section depth1 show">
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./domains.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">llms.txt in Different Domains</span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./nbdev.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">How to help LLMs understand your nbdev project</span></a>
</div>
</li>
</ul>
</li>
</ul>
</div>
</nav>
<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
<!-- margin-sidebar -->
<div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
<nav id="TOC" role="doc-toc" class="toc-active">
<h2 id="toc-title">On this page</h2>
<ul>
<li><a href="#background" id="toc-background" class="nav-link active" data-scroll-target="#background">Background</a></li>
<li><a href="#proposal" id="toc-proposal" class="nav-link" data-scroll-target="#proposal">Proposal</a></li>
<li><a href="#format" id="toc-format" class="nav-link" data-scroll-target="#format">Format</a></li>
<li><a href="#existing-standards" id="toc-existing-standards" class="nav-link" data-scroll-target="#existing-standards">Existing standards</a></li>
<li><a href="#example" id="toc-example" class="nav-link" data-scroll-target="#example">Example</a></li>
<li><a href="#directories" id="toc-directories" class="nav-link" data-scroll-target="#directories">Directories</a></li>
<li><a href="#integrations" id="toc-integrations" class="nav-link" data-scroll-target="#integrations">Integrations</a></li>
<li><a href="#next-steps" id="toc-next-steps" class="nav-link" data-scroll-target="#next-steps">Next steps</a></li>
</ul>
<div class="toc-actions"><ul><li><a href="https://github.com/AnswerDotAI/llms-txt/issues/new" class="toc-action"><i class="bi bi-github"></i>Report an issue</a></li></ul></div><div class="quarto-alternate-formats"><h2>Other Formats</h2><ul><li><a href="index.md"><i class="bi bi-file-code"></i>Markdown</a></li></ul></div></nav>
</div>
<!-- main -->
<main class="content" id="quarto-document-content">
<header id="title-block-header" class="quarto-title-block default">
<div class="quarto-title">
<h1 class="title">The /llms.txt file, v2</h1>
</div>
<div>
<div class="description">
A proposal to standardise on using an <code>/llms.txt</code> file to provide information to help agents use a website.
</div>
</div>
<div class="quarto-title-meta">
<div>
<div class="quarto-title-meta-heading">Author</div>
<div class="quarto-title-meta-contents">
<p>Jeremy Howard </p>
</div>
</div>
<div>
<div class="quarto-title-meta-heading">Published</div>
<div class="quarto-title-meta-contents">
<p class="date">September 3, 2024</p>
</div>
</div>
<div>
<div class="quarto-title-meta-heading">Modified</div>
<div class="quarto-title-meta-contents">
<p class="date-modified">August 10, 2026</p>
</div>
</div>
</div>
</header>
<section id="background" class="level2">
<h2 class="anchored" data-anchor-id="background">Background</h2>
<p>Agents now use websites constantly: a coding agent fetches a library’s documentation to get an API call right, and a chat assistant with search reads pages to answer questions about a product. When this proposal was first written in 2024, this was largely a prediction. Today it is routine.</p>
<p>But web pages are built for people. An HTML page wraps its information in navigation, ads, and JavaScript, and converting it back into clean text is difficult and imprecise. Context windows, while larger than they were, are still too small for most websites in their entirety, and every wasted token costs time and money. Agents are best served by concise, expert-level information gathered in a single, accessible location.</p>
<p>This is v2 of the proposal, updated based on what I learned from two years of adoption: thousands of sites publish an llms.txt file, <a href="https://www.mintlify.com/docs/ai/llmstxt">documentation platforms generate one automatically</a>, and Chrome’s Lighthouse <a href="https://developer.chrome.com/docs/lighthouse/agentic-browsing/llms-txt">audits sites for one</a> as part of its agentic browsing checks. The AI labs themselves publish llms.txt files for their own developer docs: <a href="https://developers.openai.com/llms.txt">OpenAI</a>, <a href="https://docs.anthropic.com/llms.txt">Anthropic</a>, and <a href="https://ai.google.dev/gemini-api/docs/llms.txt">Gemini</a>. The <a href="./changes.html">Changes</a> page describes what changed since v1, and why.</p>
</section>
<section id="proposal" class="level2">
<h2 class="anchored" data-anchor-id="proposal">Proposal</h2>
<div class="quarto-figure quarto-figure-center">
<figure class="figure">
<p><a href="logo.png" class="lightbox" data-gallery="quarto-lightbox-gallery-1" title="llms.txt logo"><img src="logo.png" class="floatr img-fluid figure-img" width="150" alt="llms.txt logo"></a></p>
<figcaption>llms.txt logo</figcaption>
</figure>
</div>
<p>We propose adding a <code>/llms.txt</code> markdown file to websites to provide LLM-friendly content. The file can be placed at the site root, or at any path within it, covering the pages under that path. This file offers brief background information, guidance, and links to detailed markdown files.</p>
<p>llms.txt markdown is human and LLM readable, but is also in a precise format allowing fixed processing methods (i.e.&nbsp;classical programming techniques such as parsers and regex).</p>
<p>We furthermore propose that pages with information that agents might need provide a clean markdown version of those pages at the same URL as the original page, either with <code>.md</code> appended (<code>page.html.md</code>) or with the extension replaced by <code>.md</code> (<code>page.md</code>). (URLs without file names should append <code>index.html.md</code> or <code>index.md</code> instead.)</p>
<p>To help clients find these files, we recommend using standard link relations: <code>rel="alternate" type="text/markdown"</code> points to the markdown version of a page, and <code>rel="describedby"</code> points to the llms.txt file that covers it. (An llms.txt file describes all pages under its path, so <code>/docs/llms.txt</code> covers everything in <code>/docs/</code>.) These links can be provided as HTML <code>&lt;link&gt;</code> elements, or as an HTTP <code>Link:</code> response header. The header form also works for non-HTML resources, such as the markdown files themselves, and can be added in web server or CDN configuration without modifying any pages. For example:</p>
<pre><code>Link: &lt;/docs/page.html.md&gt;; rel="alternate"; type="text/markdown", &lt;/docs/llms.txt&gt;; rel="describedby"</code></pre>
<p>The <a href="https://fastht.ml">FastHTML project</a> follows these two proposals for its documentation. For instance, here is the <a href="https://www.fastht.ml/docs/llms.txt">FastHTML docs llms.txt</a>, placed at <code>/docs/</code> to cover just the documentation pages. And here is an example of a <a href="https://www.fastht.ml/docs/tutorials/by_example.html">regular HTML docs page</a>, along with exact same URL but with <a href="https://www.fastht.ml/docs/tutorials/by_example.html.md">a .md extension</a>.</p>
<p>Agents are expected to view or search <code>llms.txt</code> to find the information they need, then follow the relevant links. The links in an llms.txt file should therefore point to LLM-friendly content, such as the markdown versions of pages described above. The file itself stays small enough to fit in context. The detail lives behind the links, and is fetched only when needed.</p>
<p>llms.txt files are used most heavily for software documentation, where coding agents follow them to find API references and tutorials. The same structure works anywhere agents need a guided path into a site’s content: a business outlining its structure and policies, a personal site answering questions about someone’s CV, or a school providing access to course information.</p>
<p>Note that all <a href="https://nbdev.fast.ai/">nbdev</a> projects now create .md versions of all pages by default. All Answer.AI and fast.ai software projects using nbdev have had their docs regenerated with this feature. For an example, see the <a href="https://fastcore.fast.ai/docments.html.md">markdown version</a> of <a href="https://fastcore.fast.ai/docments.html">fastcore’s docments module</a>.</p>
</section>
<section id="format" class="level2">
<h2 class="anchored" data-anchor-id="format">Format</h2>
<p>At the moment the most widely and easily understood format for language models is Markdown. Simply showing where key Markdown files can be found is a great first step. Providing some basic structure helps a language model to find where the information it needs can come from.</p>
<p>The <code>llms.txt</code> file is unusual in that it uses Markdown to structure the information rather than a classic structured format such as XML. The reason for this is that we expect many of these files to be read by language models and agents. Having said that, the information in llms.txt follows a specific format and can be read using standard programmatic-based tools.</p>
<p>The llms.txt file spec is for files named <code>llms.txt</code>, at the root path <code>/llms.txt</code> of a website or at any subpath (e.g.&nbsp;<code>/docs/llms.txt</code>). A file covers the URLs under its path, and where more than one file applies, agents should use the most specific one. A file following the spec contains the following sections as markdown, in the specific order:</p>
<ul>
<li>An optional byte-order mark (BOM)</li>
<li>An H1 with the name of the project or site. This is the only required section</li>
<li>A blockquote with a short summary of the project, containing key information necessary for understanding the rest of the file</li>
<li>Zero or more markdown sections (e.g.&nbsp;paragraphs, lists, etc) of any type except headings, containing more detailed information about the project and how to interpret the provided files</li>
<li>Zero or more markdown sections delimited by H2 headers, containing “file lists” of URLs where further detail is available
<ul>
<li>Each “file list” is a markdown list, containing a required markdown hyperlink <code>[name](url)</code>, then optionally a <code>:</code> and notes about the file.</li>
</ul></li>
</ul>
<p>Here is a mock example:</p>
<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb2"><pre class="sourceCode markdown code-with-copy"><code class="sourceCode markdown"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="fu"># Title</span></span>
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="at">&gt; Optional description goes here</span></span>
<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a>Optional details go here</span>
<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb2-7"><a href="#cb2-7" aria-hidden="true" tabindex="-1"></a><span class="fu">## Section name</span></span>
<span id="cb2-8"><a href="#cb2-8" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb2-9"><a href="#cb2-9" aria-hidden="true" tabindex="-1"></a><span class="ss">- </span><span class="co">[</span><span class="ot">Link title</span><span class="co">](https://link_url)</span>: Optional link details</span>
<span id="cb2-10"><a href="#cb2-10" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb2-11"><a href="#cb2-11" aria-hidden="true" tabindex="-1"></a><span class="fu">## Optional</span></span>
<span id="cb2-12"><a href="#cb2-12" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb2-13"><a href="#cb2-13" aria-hidden="true" tabindex="-1"></a><span class="ss">- </span><span class="co">[</span><span class="ot">Link title</span><span class="co">](https://link_url)</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
<p>The “Optional” section is used, by convention, for secondary information: links an agent can skip when a shorter context is needed.</p>
</section>
<section id="existing-standards" class="level2">
<h2 class="anchored" data-anchor-id="existing-standards">Existing standards</h2>
<p>llms.txt is designed to coexist with current web standards. While sitemaps list all pages for search engines, <code>llms.txt</code> offers a curated overview for LLMs. It can complement robots.txt by providing context for allowed content. The file can also reference structured data markup used on the site, helping LLMs understand how to interpret this information in context.</p>
<p>The approach of using a standard filename follows <code>/robots.txt</code> and <code>/sitemap.xml</code> at the site root. And just as <code>index.html</code> gives any path a conventional location for its human-readable entry point, <code>llms.txt</code> gives any path a conventional location for its LLM-readable overview. robots.txt and <code>llms.txt</code> have different purposes. robots.txt lets automated tools know what access to a site is considered acceptable, such as for search indexing bots. llms.txt information is instead used on demand, when an agent needs information about a topic while assisting a user. Our expectation was that llms.txt would mainly be useful for <em>inference</em> rather than <em>training</em>, and that is how it has been used, though training runs could take advantage of the information too.</p>
<p>An alternative would be the Well-Known URIs standard (RFC 8615), which reserves the <code>/.well-known/</code> prefix for metadata files like this one. But well-known URIs exist only at the origin root, and many authors control only a path on a shared host: a GitHub Pages project site, for example, can publish files in its own directory but can never add one to the host’s <code>/.well-known/</code>. Like <code>index.html</code>, an <code>llms.txt</code> describes the path where it sits, something a single root location cannot express. And anyone who can publish content at a path can provide one.</p>
<p>sitemap.xml is a list of all the indexable human-readable information available on a site. This isn’t a substitute for <code>llms.txt</code> since it:</p>
<ul>
<li>Often won’t have the LLM-readable versions of pages listed</li>
<li>Doesn’t include URLs to external sites, even though they might be helpful to understand the information</li>
<li>Will generally cover documents that in aggregate will be too large to fit in an LLM context window, and will include a lot of information that isn’t necessary to understand the site.</li>
</ul>
</section>
<section id="example" class="level2">
<h2 class="anchored" data-anchor-id="example">Example</h2>
<p>Here’s an example of <code>llms.txt</code>, in this case a cut down version of the file used for the FastHTML project (see also the <a href="https://www.fastht.ml/docs/llms.txt">full version</a>):</p>
<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb3"><pre class="sourceCode markdown code-with-copy"><code class="sourceCode markdown"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="fu"># FastHTML</span></span>
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="at">&gt; FastHTML is a python library which brings together Starlette, Uvicorn, HTMX, and fastcore's </span><span class="in">`FT`</span><span class="at"> "FastTags" into a library for creating server-rendered hypermedia applications.</span></span>
<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb3-5"><a href="#cb3-5" aria-hidden="true" tabindex="-1"></a>Important notes:</span>
<span id="cb3-6"><a href="#cb3-6" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb3-7"><a href="#cb3-7" aria-hidden="true" tabindex="-1"></a><span class="ss">- </span>Although parts of its API are inspired by FastAPI, it is *not* compatible with FastAPI syntax and is not targeted at creating API services</span>
<span id="cb3-8"><a href="#cb3-8" aria-hidden="true" tabindex="-1"></a><span class="ss">- </span>FastHTML is compatible with JS-native web components and any vanilla JS library, but not with React, Vue, or Svelte.</span>
<span id="cb3-9"><a href="#cb3-9" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb3-10"><a href="#cb3-10" aria-hidden="true" tabindex="-1"></a><span class="fu">## Docs</span></span>
<span id="cb3-11"><a href="#cb3-11" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb3-12"><a href="#cb3-12" aria-hidden="true" tabindex="-1"></a><span class="ss">- </span><span class="co">[</span><span class="ot">FastHTML quick start</span><span class="co">](https://fastht.ml/docs/tutorials/quickstart_for_web_devs.html.md)</span>: A brief overview of many FastHTML features</span>
<span id="cb3-13"><a href="#cb3-13" aria-hidden="true" tabindex="-1"></a><span class="ss">- </span><span class="co">[</span><span class="ot">HTMX reference</span><span class="co">](https://github.com/bigskysoftware/htmx/blob/master/www/content/reference.md)</span>: Brief description of all HTMX attributes, CSS classes, headers, events, extensions, js lib methods, and config options</span>
<span id="cb3-14"><a href="#cb3-14" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb3-15"><a href="#cb3-15" aria-hidden="true" tabindex="-1"></a><span class="fu">## Examples</span></span>
<span id="cb3-16"><a href="#cb3-16" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb3-17"><a href="#cb3-17" aria-hidden="true" tabindex="-1"></a><span class="ss">- </span><span class="co">[</span><span class="ot">Todo list application</span><span class="co">](https://github.com/AnswerDotAI/fasthtml/blob/main/examples/adv_app.py)</span>: Detailed walk-thru of a complete CRUD app in FastHTML showing idiomatic use of FastHTML and HTMX patterns.</span>
<span id="cb3-18"><a href="#cb3-18" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb3-19"><a href="#cb3-19" aria-hidden="true" tabindex="-1"></a><span class="fu">## Optional</span></span>
<span id="cb3-20"><a href="#cb3-20" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb3-21"><a href="#cb3-21" aria-hidden="true" tabindex="-1"></a><span class="ss">- </span><span class="co">[</span><span class="ot">Starlette full documentation</span><span class="co">](https://gist.githubusercontent.com/jph00/809e4a4808d4510be0e3dc9565e9cbd3/raw/9b717589ca44cedc8aaf00b2b8cacef922964c0f/starlette-sml.md)</span>: A subset of the Starlette documentation useful for FastHTML development. </span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
<p>To create effective <code>llms.txt</code> files, consider these guidelines:</p>
<ul>
<li>Use concise, clear language.</li>
<li>When linking to resources, include brief, informative descriptions.</li>
<li>Avoid ambiguous terms or unexplained jargon.</li>
<li>Test your file by asking an agent questions about your content, giving it only your llms.txt as a starting point.</li>
</ul>
</section>
<section id="directories" class="level2">
<h2 class="anchored" data-anchor-id="directories">Directories</h2>
<p>Here are a few directories that list the <code>llms.txt</code> files available on the web:</p>
<ul>
<li><a href="https://llmstxt.site/">llmstxt.site</a></li>
<li><a href="https://directory.llmstxt.cloud/">directory.llmstxt.cloud</a></li>
<li><a href="https://llmstxthub.com/">llmstxthub.com</a></li>
</ul>
</section>
<section id="integrations" class="level2">
<h2 class="anchored" data-anchor-id="integrations">Integrations</h2>
<p>Many documentation platforms and CMSs can generate an llms.txt file automatically:</p>
<ul>
<li><a href="https://www.mintlify.com/docs/ai/llmstxt">Mintlify</a> - Docs platform that generates llms.txt and markdown page versions for every site it hosts</li>
<li><a href="https://www.gitbook.com/blog/what-is-llms-txt">GitBook</a> - Serves an llms.txt file for published docs sites</li>
<li><a href="https://yoast.com/features/llms-txt/">Yoast SEO</a> - WordPress plugin that generates and maintains an llms.txt file</li>
<li><a href="https://aioseo.com/features/llms-txt/">AIOSEO</a> - WordPress plugin with an llms.txt generator</li>
<li><a href="https://support.wix.com/en/article/understanding-your-sites-llmstxt-file">Wix</a> - Generates an llms.txt file for every Wix site</li>
</ul>
<p>And various libraries and plugins are available to integrate the llms.txt specification into your workflow:</p>
<ul>
<li><a href="./llmstxt-js.html">JavaScript Implementation</a> - Sample JavaScript implementation</li>
<li><a href="https://github.com/okineadev/vitepress-plugin-llms"><code>vitepress-plugin-llms</code></a> - VitePress plugin that automatically generates LLM-friendly documentation for the website following the llms.txt specification</li>
<li><a href="https://github.com/rachfop/docusaurus-plugin-llms"><code>docusaurus-plugin-llms</code></a> - Docusaurus plugin for generating LLM-friendly documentation following the llmtxt.org standard</li>
<li><a href="https://www.drupal.org/project/llm_support">Drupal LLM Support</a> - A Drupal Recipe providing full support for the llms.txt proposal on any Drupal 10.3+ site</li>
<li><a href="https://github.com/raphaelstolt/llms-txt-php"><code>llms-txt-php</code></a> - A library for writing and reading llms.txt Markdown files</li>
<li><a href="https://dmux.github.io/pagepilot"><code>VS Code PagePilot Extension</code></a> - PagePilot is a VS Code Chat participant that automatically loads external context (documentation, APIs, README files) to provide enhanced responses.</li>
<li><a href="https://github.com/mcp-get/community-servers/tree/main/src/server-llm-txt"><code>server-llm-txt</code></a> - MCP server that lets agents fetch and search llms.txt files</li>
</ul>
</section>
<section id="next-steps" class="level2">
<h2 class="anchored" data-anchor-id="next-steps">Next steps</h2>
<p>The <code>llms.txt</code> specification is open for community input. A <a href="https://github.com/AnswerDotAI/llms-txt">GitHub repository</a> hosts <a href="https://github.com/AnswerDotAI/llms-txt/blob/main/nbs/index.qmd">this informal overview</a>, allowing for version control and public discussion. A <a href="https://discord.gg/aJPygMvPEN">community discord channel</a> is available for sharing implementation experiences and discussing best practices.</p>
</section>
</main> <!-- /main -->
<script id="quarto-html-after-body" type="application/javascript">
window.document.addEventListener("DOMContentLoaded", function (event) {
const icon = "";
const anchorJS = new window.AnchorJS();
anchorJS.options = {
placement: 'right',
icon: icon
};
anchorJS.add('.anchored');
const isCodeAnnotation = (el) => {
for (const clz of el.classList) {
if (clz.startsWith('code-annotation-')) {
return true;
}
}
return false;
}
const onCopySuccess = function(e) {
// button target
const button = e.trigger;
// don't keep focus
button.blur();
// flash "checked"
button.classList.add('code-copy-button-checked');
var currentTitle = button.getAttribute("title");
button.setAttribute("title", "Copied!");
let tooltip;
if (window.bootstrap) {
button.setAttribute("data-bs-toggle", "tooltip");
button.setAttribute("data-bs-placement", "left");
button.setAttribute("data-bs-title", "Copied!");
tooltip = new bootstrap.Tooltip(button,
{ trigger: "manual",
customClass: "code-copy-button-tooltip",
offset: [0, -8]});
tooltip.show();
}
setTimeout(function() {
if (tooltip) {
tooltip.hide();
button.removeAttribute("data-bs-title");
button.removeAttribute("data-bs-toggle");
button.removeAttribute("data-bs-placement");
}
button.setAttribute("title", currentTitle);
button.classList.remove('code-copy-button-checked');
}, 1000);
// clear code selection
e.clearSelection();
}
const getTextToCopy = function(trigger) {
const outerScaffold = trigger.parentElement.cloneNode(true);
const codeEl = outerScaffold.querySelector('code');
for (const childEl of codeEl.children) {
if (isCodeAnnotation(childEl)) {
childEl.remove();
}
}
return codeEl.innerText;
}
const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
text: getTextToCopy
});
clipboard.on('success', onCopySuccess);
if (window.document.getElementById('quarto-embedded-source-code-modal')) {
const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
text: getTextToCopy,
container: window.document.getElementById('quarto-embedded-source-code-modal')
});
clipboardModal.on('success', onCopySuccess);
}
var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
var mailtoRegex = new RegExp(/^mailto:/);
var filterRegex = new RegExp("https:\/\/llmstxt\.org\/");
var isInternal = (href) => {
return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
}
// Inspect non-navigation links and adorn them if external
var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
for (var i=0; i<links.length; i++) {
const link = links[i];
if (!isInternal(link.href)) {
// undo the damage that might have been done by quarto-nav.js in the case of
// links that we want to consider external
if (link.dataset.originalHref !== undefined) {
link.href = link.dataset.originalHref;
}
}
}
function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
const config = {
allowHTML: true,
maxWidth: 500,
delay: 100,
arrow: false,
appendTo: function(el) {
return el.parentElement;
},
interactive: true,
interactiveBorder: 10,
theme: 'quarto',
placement: 'bottom-start',
};
if (contentFn) {
config.content = contentFn;
}
if (onTriggerFn) {
config.onTrigger = onTriggerFn;
}
if (onUntriggerFn) {
config.onUntrigger = onUntriggerFn;
}
window.tippy(el, config);
}
const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
for (var i=0; i<noterefs.length; i++) {
const ref = noterefs[i];
tippyHover(ref, function() {
// use id or data attribute instead here
let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
try { href = new URL(href).hash; } catch {}
const id = href.replace(/^#\/?/, "");
const note = window.document.getElementById(id);
if (note) {
return note.innerHTML;
} else {
return "";
}
});
}
const xrefs = window.document.querySelectorAll('a.quarto-xref');
const processXRef = (id, note) => {
// Strip column container classes
const stripColumnClz = (el) => {
el.classList.remove("page-full", "page-columns");
if (el.children) {
for (const child of el.children) {
stripColumnClz(child);
}
}
}
stripColumnClz(note)
if (id === null || id.startsWith('sec-')) {
// Special case sections, only their first couple elements
const container = document.createElement("div");
if (note.children && note.children.length > 2) {
container.appendChild(note.children[0].cloneNode(true));
for (let i = 1; i < note.children.length; i++) {
const child = note.children[i];
if (child.tagName === "P" && child.innerText === "") {
continue;
} else {
container.appendChild(child.cloneNode(true));
break;
}
}
if (window.Quarto?.typesetMath) {
window.Quarto.typesetMath(container);
}
return container.innerHTML
} else {
if (window.Quarto?.typesetMath) {
window.Quarto.typesetMath(note);
}
return note.innerHTML;
}
} else {
// Remove any anchor links if they are present
const anchorLink = note.querySelector('a.anchorjs-link');
if (anchorLink) {
anchorLink.remove();
}
if (window.Quarto?.typesetMath) {
window.Quarto.typesetMath(note);
}
if (note.classList.contains("callout")) {
return note.outerHTML;
} else {
return note.innerHTML;
}
}
}
for (var i=0; i<xrefs.length; i++) {
const xref = xrefs[i];
tippyHover(xref, undefined, function(instance) {
instance.disable();
let url = xref.getAttribute('href');
let hash = undefined;
if (url.startsWith('#')) {
hash = url;
} else {
try { hash = new URL(url).hash; } catch {}
}
if (hash) {
const id = hash.replace(/^#\/?/, "");
const note = window.document.getElementById(id);
if (note !== null) {
try {
const html = processXRef(id, note.cloneNode(true));
instance.setContent(html);
} finally {
instance.enable();
instance.show();
}
} else {
// See if we can fetch this
fetch(url.split('#')[0])
.then(res => res.text())
.then(html => {
const parser = new DOMParser();
const htmlDoc = parser.parseFromString(html, "text/html");
const note = htmlDoc.getElementById(id);
if (note !== null) {
const html = processXRef(id, note);
instance.setContent(html);
}
}).finally(() => {
instance.enable();
instance.show();
});
}
} else {
// See if we can fetch a full url (with no hash to target)
// This is a special case and we should probably do some content thinning / targeting
fetch(url)
.then(res => res.text())
.then(html => {
const parser = new DOMParser();
const htmlDoc = parser.parseFromString(html, "text/html");
const note = htmlDoc.querySelector('main.content');
if (note !== null) {
// This should only happen for chapter cross references
// (since there is no id in the URL)
// remove the first header
if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
note.children[0].remove();
}
const html = processXRef(null, note);
instance.setContent(html);
}
}).finally(() => {
instance.enable();
instance.show();
});
}
}, function(instance) {
});
}
let selectedAnnoteEl;
const selectorForAnnotation = ( cell, annotation) => {
let cellAttr = 'data-code-cell="' + cell + '"';
let lineAttr = 'data-code-annotation="' + annotation + '"';
const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
return selector;
}
const selectCodeLines = (annoteEl) => {
const doc = window.document;
const targetCell = annoteEl.getAttribute("data-target-cell");
const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
const lines = annoteSpan.getAttribute("data-code-lines").split(",");
const lineIds = lines.map((line) => {
return targetCell + "-" + line;
})
let top = null;
let height = null;
let parent = null;
if (lineIds.length > 0) {
//compute the position of the single el (top and bottom and make a div)
const el = window.document.getElementById(lineIds[0]);
top = el.offsetTop;
height = el.offsetHeight;
parent = el.parentElement.parentElement;
if (lineIds.length > 1) {
const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
const bottom = lastEl.offsetTop + lastEl.offsetHeight;
height = bottom - top;
}
if (top !== null && height !== null && parent !== null) {
// cook up a div (if necessary) and position it
let div = window.document.getElementById("code-annotation-line-highlight");
if (div === null) {
div = window.document.createElement("div");
div.setAttribute("id", "code-annotation-line-highlight");
div.style.position = 'absolute';
parent.appendChild(div);
}
div.style.top = top - 2 + "px";
div.style.height = height + 4 + "px";
div.style.left = 0;
let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
if (gutterDiv === null) {
gutterDiv = window.document.createElement("div");
gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
gutterDiv.style.position = 'absolute';
const codeCell = window.document.getElementById(targetCell);
const gutter = codeCell.querySelector('.code-annotation-gutter');
gutter.appendChild(gutterDiv);
}
gutterDiv.style.top = top - 2 + "px";
gutterDiv.style.height = height + 4 + "px";
}
selectedAnnoteEl = annoteEl;
}
};
const unselectCodeLines = () => {
const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
elementsIds.forEach((elId) => {
const div = window.document.getElementById(elId);
if (div) {
div.remove();
}
});
selectedAnnoteEl = undefined;
};
// Handle positioning of the toggle
window.addEventListener(
"resize",
throttle(() => {
elRect = undefined;
if (selectedAnnoteEl) {
selectCodeLines(selectedAnnoteEl);
}
}, 10)
);
function throttle(fn, ms) {
let throttle = false;
let timer;
return (...args) => {
if(!throttle) { // first call gets through
fn.apply(this, args);
throttle = true;
} else { // all the others get throttled
if(timer) clearTimeout(timer); // cancel #2
timer = setTimeout(() => {
fn.apply(this, args);
timer = throttle = false;
}, ms);
}
};
}
// Attach click handler to the DT
const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
for (const annoteDlNode of annoteDls) {
annoteDlNode.addEventListener('click', (event) => {
const clickedEl = event.target;
if (clickedEl !== selectedAnnoteEl) {
unselectCodeLines();
const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
if (activeEl) {
activeEl.classList.remove('code-annotation-active');
}
selectCodeLines(clickedEl);
clickedEl.classList.add('code-annotation-active');
} else {
// Unselect the line
unselectCodeLines();
clickedEl.classList.remove('code-annotation-active');
}
});
}
const findCites = (el) => {
const parentEl = el.parentElement;
if (parentEl) {
const cites = parentEl.dataset.cites;
if (cites) {
return {
el,
cites: cites.split(' ')
};
} else {
return findCites(el.parentElement)
}
} else {
return undefined;
}
};
var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
for (var i=0; i<bibliorefs.length; i++) {
const ref = bibliorefs[i];
const citeInfo = findCites(ref);
if (citeInfo) {
tippyHover(citeInfo.el, function() {
var popup = window.document.createElement('div');
citeInfo.cites.forEach(function(cite) {
var citeDiv = window.document.createElement('div');
citeDiv.classList.add('hanging-indent');
citeDiv.classList.add('csl-entry');
var biblioDiv = window.document.getElementById('ref-' + cite);
if (biblioDiv) {
citeDiv.innerHTML = biblioDiv.innerHTML;
}
popup.appendChild(citeDiv);
});
return popup.innerHTML;
});
}
}
});
</script>
</div> <!-- /content -->
<footer class="footer"><div class="nav-footer"><div class="nav-footer-center"><div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/AnswerDotAI/llms-txt/issues/new" class="toc-action"><i class="bi bi-github"></i>Report an issue</a></li></ul></div></div></div></footer><script>var lightboxQuarto = GLightbox({"closeEffect":"zoom","descPosition":"bottom","loop":false,"openEffect":"zoom","selector":".lightbox"});
(function() {
let previousOnload = window.onload;
window.onload = () => {
if (previousOnload) {
previousOnload();
}
lightboxQuarto.on('slide_before_load', (data) => {
const { slideIndex, slideNode, slideConfig, player, trigger } = data;
const href = trigger.getAttribute('href');
if (href !== null) {
const imgEl = window.document.querySelector(`a[href="${href}"] img`);
if (imgEl !== null) {
const srcAttr = imgEl.getAttribute("src");
if (srcAttr && srcAttr.startsWith("data:")) {
slideConfig.href = srcAttr;
}
}
}
});
lightboxQuarto.on('slide_after_load', (data) => {
const { slideIndex, slideNode, slideConfig, player, trigger } = data;
if (window.Quarto?.typesetMath) {
window.Quarto.typesetMath(slideNode);
}
});
};
})();
</script>
</body></html>