examples/markdown/

# The document: a list of blocks, read one line at a time.
#
# Markdown's block grammar is line-oriented, so this is a `while` over the lines
# with an index rather than a `for` — a fenced code block consumes lines the loop
# has not reached yet, and `for` gives no way to say so.
#
# What decides a line's kind is its **first characters after the indentation**,
# and the order of the tests below is part of the grammar: `***` is a horizontal
# rule and `* item` is a bullet, so the rule is asked about first.

use inline

variant Block
    heading
        level: i64
        text: str
    paragraph
        text: str
    bullet
        text: str
    numbered
        n: i64
        text: str
    quote
        text: str
    code
        body: [str]
    rule

## Reading lines

# The text, split at newlines. A trailing newline does not make a last empty
# line, which is the convention every `read_file` in this corpus relies on.
function split_lines(text: str) -> [str]
    out: [str] @= []
    start: i64 @= 0
    i: i64 @= 0

    while i < text.len()
        if text[i] == '\n'
            out @ out.push(text.slice(from: start, to: i))
            start @ i + 1
        i @ i + 1

    if start < text.len()
        out @ out.push(text.slice(from: start, to: text.len()))
    return out

function is_space(c: u8) -> bool
    return c == ' ' || c == '\t' || c == 13

# The line without its surrounding whitespace.
function trimmed(line: str) -> str
    from: i64 @= 0

    while from < line.len() && is_space(line[from])
        from @ from + 1

    to: i64 @= line.len()

    while to > from && is_space(line[to - 1])
        to @ to - 1

    return line.slice(from: from, to: to)

function starts(line: str, word: str) -> bool
    if word.len() > line.len()
        return false
    return line.slice(from: 0, to: word.len()) == word

# How many times the line opens with `c`.
function leading(line: str, c: u8) -> i64
    n: i64 @= 0

    while n < line.len() && line[n] == c
        n @ n + 1

    return n

function digits_of(line: str) -> i64
    n: i64 @= 0

    while n < line.len() && line[n] >= '0' && line[n] <= '9'
        n @ n + 1

    return n

# `---`, `***` or `___`, three or more, and nothing else on the line.
function is_rule(head: str) -> bool
    if head.len() < 3
        return false
    c = head[0]

    if c != '-' && c != '*' && c != '_'
        return false
    return leading(head, c) == head.len()

# `12. ` — the width of the marker, or 0 for a line that is not a numbered item.
function marker_width(head: str) -> i64
    n = digits_of(head)

    if n == 0 || n + 1 >= head.len()
        return 0

    if head[n] != '.' || head[n + 1] != ' '
        return 0
    return n

function number_of(head: str, width: i64) -> i64
    v: i64 @= 0

    for i in range(from: 0, to: width)
        v @ v * 10 + to_i64(head[i] - '0').must()

    return v

function after(head: str, n: i64) -> str
    return trimmed(head.slice(from: n, to: head.len()))

# Whether a line ends the paragraph above it. Every block opener does, and so
# does a blank line — which is the only reason a paragraph ever ends at all.
function opens_a_block(head: str) -> bool
    if head == "" || head.starts(word: "```") || is_rule(head)
        return true

    if head.starts(word: "#") || head.starts(word: "> ") || head.starts(word: "- ") || head.starts(word: "* ")
        return true
    return marker_width(head) > 0

## The reader

function read(text: str) -> [Block]
    lines = split_lines(text)
    out: [Block] @= []
    i: i64 @= 0

    while i < lines.len()
        head = trimmed(lines[i])

        if head.starts(word: "```")
            body: [str] @= []
            i @ i + 1

            while i < lines.len() && !trimmed(lines[i]).starts(word: "```")
                body @ body.push(lines[i])
                i @ i + 1

            if i < lines.len()
                i @ i + 1
            out @ out.push(.code(body: body))
        else if is_rule(head)
            out @ out.push(.rule)
            i @ i + 1
        else if head == ""
            i @ i + 1
        else if head.starts(word: "#")
            level = leading(head, '#')
            out @ out.push(.heading(level: level, text: inline.plain(after(head, level))))
            i @ i + 1
        else if head.starts(word: "> ")
            out @ out.push(.quote(text: inline.plain(after(head, 2))))
            i @ i + 1
        else if head.starts(word: "- ") || head.starts(word: "* ")
            out @ out.push(.bullet(text: inline.plain(after(head, 2))))
            i @ i + 1
        else if marker_width(head) > 0
            width = marker_width(head)
            out @ out.push(.numbered(n: number_of(head, width), text: inline.plain(after(head, width + 1))))
            i @ i + 1
        else
            parts: [str] @= [inline.plain(head)]
            i @ i + 1

            while i < lines.len() && !opens_a_block(trimmed(lines[i]))
                parts @ parts.push(inline.plain(trimmed(lines[i])))
                i @ i + 1

            out @ out.push(.paragraph(text: join(parts, " ")))

    return out

test "the line kinds are told apart by what opens them"
    blocks = read("# Title\n\ntext\n\n- one\n- two\n\n1. first\n\n> quoted\n\n---\n")
    assert blocks.len() == 7
    assert blocks[0] == .heading(level: 1, text: "Title")
    assert blocks[1] == .paragraph(text: "text")
    assert blocks[2] == .bullet(text: "one")
    assert blocks[4] == .numbered(n: 1, text: "first")
    assert blocks[5] == .quote(text: "quoted")
    assert blocks[6] == .rule

test "a rule is asked about before a bullet, because `***` is both"
    only_rule: [Block] = [.rule]
    only_bullet: [Block] = [.bullet(text: "item")]
    assert read("***\n") == only_rule
    assert read("* item\n") == only_bullet

test "a paragraph runs until something opens a block"
    blocks = read("one\ntwo\n# stop\n")
    assert blocks.len() == 2
    assert blocks[0] == .paragraph(text: "one two")

test "a fence takes its lines exactly as they are"
    verbatim: [Block] = [.code(body: ["  x = 1", "**not bold**"])]
    assert read("```\n  x = 1\n**not bold**\n```\n") == verbatim

test "an unclosed fence takes the rest of the document"
    rest: [Block] = [.code(body: ["a", "b"])]
    assert read("```\na\nb\n") == rest

test "a heading counts its hashes"
    deep: [Block] = [.heading(level: 3, text: "Deep")]
    assert read("### Deep\n") == deep

test "split_lines does not invent a last empty line"
    assert split_lines("a\nb\n") == ["a", "b"]
    assert split_lines("a\nb") == ["a", "b"]
    assert split_lines("") == []

# Inline markup, removed.
#
# The innermost module of the four, and the one with no `use` line: it knows
# about a line of text and nothing about documents. `**bold**` and `*italic*`
# lose their stars, `` `code` `` loses its backticks, and `[label](url)` becomes
# `label (url)` — the link's target kept rather than dropped, because plain text
# is where a reader can no longer click it.
#
# `_` is deliberately **not** an italic marker here. It is the one that cannot be
# told from an identifier without more context than a line has, and stripping it
# would quietly rewrite `to_str` as `tostr` in every sentence about code.

record Scan
    text: str
    pos: i64

function at_end(s: Scan) -> bool
    return s.pos >= s.text.len()

function here(s: Scan) -> u8
    return s.text[s.pos]

function advance(@s: Scan)
    s.pos @ s.pos + 1

function skip(@s: Scan, n: i64)
    s.pos @ s.pos + n

# The single character under the cursor, as a `str` — `s[i]` gives a `u8`, and a
# byte is not a string.
function one(s: Scan) -> str
    return s.text.slice(from: s.pos, to: s.pos + 1)

# Whether the text from here on opens with `word`, asked without moving and
# without ever slicing past the end: an out-of-bounds slice **aborts** (§4.14).
function opens_with(s: Scan, word: str) -> bool
    stop = s.pos + word.len()

    if stop > s.text.len()
        return false
    return s.text.slice(from: s.pos, to: stop) == word

# One line of Markdown, as the text a reader would be left with.
function plain(text: str) -> str
    s: Scan @= Scan(text: text, pos: 0)
    out: [str] @= []

    while !at_end(s)
        if opens_with(s, "**")
            skip(@s, 2)
        else if here(s) == '*' || here(s) == '`'
            advance(@s)
        else if here(s) == '['
            out @ out.push(link(@s))
        else
            out @ out.push(one(s))
            advance(@s)

    return join(out, "")

# `[label](url)` — or, if what follows is not that, the `[` itself.
#
# The failure arm rewinds to one past the opening bracket rather than to the
# bracket, and that is the whole reason this reads left to right without ever
# looping: the caller resumes *after* the character it just emitted, so a `[`
# that opens nothing costs one step and not a rescan.
function link(@s: Scan) -> str
    opened = s.pos
    advance(@s)
    label: [str] @= []

    while !at_end(s) && here(s) != ']'
        label @ label.push(one(s))
        advance(@s)

    if at_end(s)
        return give_up(@s, opened)
    advance(@s)

    if at_end(s) || here(s) != '('
        return give_up(@s, opened)
    advance(@s)
    target: [str] @= []

    while !at_end(s) && here(s) != ')'
        target @ target.push(one(s))
        advance(@s)

    if at_end(s)
        return give_up(@s, opened)
    advance(@s)
    return plain(join(label, "")) + " (" + join(target, "") + ")"

function give_up(@s: Scan, opened: i64) -> str
    s.pos @ opened + 1
    return "["

test "emphasis loses its markers and keeps its words"
    assert plain("a **bold** word") == "a bold word"
    assert plain("a *thin* word") == "a thin word"
    assert plain("call `to_str` on it") == "call to_str on it"

test "an underscore is left alone, because it is also a name"
    assert plain("read `hero_runtime_check_leaks` twice") == "read hero_runtime_check_leaks twice"

test "a link keeps its target"
    assert plain("see [the spec](spec/heroes-spec.md) first") == "see the spec (spec/heroes-spec.md) first"
    assert plain("[**bold label**](x)") == "bold label (x)"

test "a bracket that opens nothing is a bracket"
    assert plain("an [unclosed label") == "an [unclosed label"
    assert plain("a [label] with no target") == "a [label] with no target"
    assert plain("a [label](unclosed") == "a [label](unclosed"

# Blocks out, plain text in — the half that decides what the document *looks*
# like once nothing is markup any more.
#
# Every block renders to a list of lines and the lines are joined once, at the
# end. That is the shape the spec's own cost paragraph asks for: `+` on `str`
# copies both sides, so building a page by concatenation is quadratic in its
# length, and `join` is linear.

use blocks

# A heading is underlined the way a typewriter would do it — `=` for the top
# level and `-` under everything else. The rule's width is the heading's own
# byte length, which is its visible width for ASCII and is what §11 asks these
# files to be.
function underline(level: i64, text: str) -> str
    if level <= 1
        return repeat("=", text.len().to_u64().must())
    return repeat("-", text.len().to_u64().must())

function indent(lines: [str], prefix: str) -> [str]
    out: [str] @= []

    for line in lines
        out @ out.push(prefix + line)

    return out

# One block, as the lines it becomes.
function lines_of(b: blocks.Block) -> [str]
    return match b
        .heading h   => [h.text, underline(h.level, h.text)]
        .paragraph p => [p.text]
        .bullet x    => ["  * " + x.text]
        .numbered x  => ["  " + x.n.to_str() + ". " + x.text]
        .quote q     => ["  | " + q.text]
        .code c      => indent(c.body, "    ")
        .rule        => [repeat("-", 60)]

# Which run of neighbours a block belongs to, as a small number. Zero is "on its
# own", and it is the answer for every block that is not a list item.
function run_kind(b: blocks.Block) -> i64
    return match b
        .bullet _                                   => 1
        .numbered _                                 => 2
        .quote _                                    => 3
        .heading _ | .paragraph _ | .code _ | .rule => 0

# Whether two neighbours belong to one run, and so take no blank line between
# them. A list is one thing on the page, and Markdown's own source writes it
# that way — three bullets with a blank line between each is three lists.
function runs_on(before: blocks.Block, after: blocks.Block) -> bool
    kind = run_kind(before)
    return kind != 0 && kind == run_kind(after)

# The whole document. Blocks are separated by one blank line unless they are one
# run, and the text ends in exactly one newline — the same promise `print` makes.
#
# An index rather than `for b in document`, because the separator is a question
# about a *pair* of blocks and the loop has to be able to look behind it.
function render(document: [blocks.Block]) -> str
    out: [str] @= []
    i: i64 @= 0

    while i < document.len()
        if i > 0 && !runs_on(before: document[i - 1], after: document[i])
            out @ out.push("")

        for line in lines_of(document[i])
            out @ out.push(line)

        i @ i + 1

    if out.len() == 0
        return ""
    return join(out, "\n") + "\n"

# The two halves in one call, which is what the program does.
function to_text(source: str) -> str
    return render(blocks.read(source))

test "a heading is underlined to its own width"
    assert to_text("# Hi\n") == "Hi\n==\n"
    assert to_text("## Hi\n") == "Hi\n--\n"

test "the list markers are rewritten rather than removed"
    assert to_text("- one\n") == "  * one\n"
    assert to_text("3. three\n") == "  3. three\n"
    assert to_text("> said\n") == "  | said\n"

test "code keeps its own layout, indented"
    assert to_text("```\n  x = 1\n```\n") == "      x = 1\n"

test "blocks are separated by one blank line"
    assert to_text("# T\n\nbody\n") == "T\n=\n\nbody\n"

test "a list is one run, and two kinds of list are two"
    assert to_text("- a\n- b\n") == "  * a\n  * b\n"
    assert to_text("1. a\n2. b\n") == "  1. a\n  2. b\n"
    assert to_text("- a\n\n1. b\n") == "  * a\n\n  1. b\n"
    assert to_text("> a\n> b\n") == "  | a\n  | b\n"

test "an empty document renders to nothing at all"
    assert to_text("") == ""
    assert to_text("\n\n\n") == ""

# **This file used to define `repeat` itself**, and it defined it the slow way —
# `push` in a loop, then `join` — which is precisely the program
# `docs/measurements/007` measured a blind reader writing, and precisely the
# quadratic the spec's cost clause names. It was here, in this project's own
# corpus, before the built-in existed to replace it (panel 054).
#
# The old version returned `""` for a negative count, and the built-in cannot be
# asked: its count is a `u64`, so a negative one does not compile. Where the
# count is computed — the two `text.len()` calls above — the conversion is
# written, and it aborts at the value that went wrong rather than inside
# `repeat`.
test "repeat builds a run of one string"
    assert repeat("x", 0) == ""
    assert repeat("ab", 3) == "ababab"
    assert repeat("-", 3) == "---"