from pathlib import Path
import re,itertools
def chunked(it, chunk_sz):
it = iter(it)
return iter(lambda: list(itertools.islice(it, chunk_sz)), [])
def parse_llms_txt(txt):
"Parse llms.txt file contents in `txt` to a `dict`"
def _p(links):
link_pat = '-\s*\[(?P
[^\]]+)\]\((?P[^\)]+)\)(?::\s*(?P.*))?'
return [re.search(link_pat, l).groupdict()
for l in re.split(r'\n+', links.strip()) if l.strip()]
start,*rest = re.split(fr'^##\s*(.*?$)', txt, flags=re.MULTILINE)
sects = {k: _p(v) for k,v in dict(chunked(rest, 2)).items()}
pat = '^#\s*(?P.+?$)\n+(?:^>\s*(?P.+?$)$)?\n+(?P.*)'
d = re.search(pat, start.strip(), (re.MULTILINE|re.DOTALL)).groupdict()
d['sections'] = sects
return d