1ac13cd | plum | pyrossh

git clone https://git.pyrossh.dev/plum

A statically typed, imperative programming language inspired by rust, python

1ac13cd — Peter John 2026-11-02T15:34:17+05:30

Report parse errors at the offending token with a hint for `()`, in the loader and LSP

Files changed (4) hide show

plum-core/src/lib.rs +3 -1
plum-core/src/loader.rs +220 -13
plum-core/tests/loader_test.rs +52 -0
plum-lsp/src/main.rs +41 -40

plum-core/src/lib.rs CHANGED Viewed

@@ -14,5 +14,7 @@ pub mod unions;
 pub use formatter::{
     formatSource, formatSourceWithOpts, inlineSingleStatementCaseBodies, FormatterError,
 };
+pub use loader::{
-pub use loader::{loadAndMerge, loadAndMergeWithOverrides};
+    loadAndMerge, loadAndMergeWithOverrides, syntaxErrors, SyntaxError, SyntaxErrorKind,
+};
 pub use parser::AstParser;

plum-core/src/loader.rs CHANGED Viewed

@@ -214,14 +214,13 @@ fn parseFileWithOverrides(
     // (including `ERROR`). Left unchecked, that meant a single bad token
     // anywhere in the file silently discarded every top-level item after
     // it — no parse error, just fewer items than the file actually has.
-    if let Some(err) = firstErrorNode(root) {
+    if let Some(err) = syntaxErrors(root, &text).into_iter().next() {
-        let start = err.start_position();
         return Err(format!(
-            "parse error in '{}' at line {}, column {}: unexpected token near '{}'",
+            "parse error in '{}' at line {}, column {}: {}",
             path.display(),
-            start.row + 1,
+            err.start.row + 1,
-            start.column + 1,
+            err.start.column + 1,
-            err.utf8_text(text.as_bytes()).unwrap_or("").trim(),
+            err.message(),
         ));
     }
     let ap = AstParser::new(&text);
@@ -233,23 +232,231 @@ fn parseFileWithOverrides(
     Ok(source)
 }
+/// A syntax error pinned to the token tree-sitter could not fit.
+#[derive(Debug, Clone, PartialEq)]
+pub struct SyntaxError {
-/// The first `ERROR` (or MISSING-token) node in `node`'s subtree, depth-first,
+    /// Start of the offending token (for end of file, the end of the last token).
+    pub start: tree_sitter::Point,
+    /// End of the offending token; equal to `start` at end of file.
+    pub end: tree_sitter::Point,
+    pub kind: SyntaxErrorKind,
+}
+#[derive(Debug, Clone, PartialEq)]
+pub enum SyntaxErrorKind {
-/// or `None` if the parse was clean. Every `ERROR`/missing node's own text
+    /// A token that fits nowhere, as it appears in the source (single line, at most 40 chars).
+    Unexpected(String),
+    /// The grammar needs this node and the source has none.
+    Missing(String),
+    /// An opening delimiter the rest of the file never closes.
+    Unclosed(String),
+    EndOfFile,
+}
+impl SyntaxError {
+    /// `unexpected '<token>'`, `expected <kind>` or `unexpected end of file`,
+    /// plus a second `hint:` line for mistakes with a known fix.
+    pub fn message(&self) -> String {
+        match &self.kind {
+            SyntaxErrorKind::Unexpected(tok) => {
+                let mut msg = format!("unexpected '{}'", tok);
+                if tok == "()" {
+                    msg.push_str(
+                        "\n  hint: `()` is not a value in Plum; a body with nothing to do \
+                         can be `_ := 0` (or `todo` while it is unimplemented)",
+                    );
+                }
+                msg
+            }
+            SyntaxErrorKind::Missing(kind) => format!("expected {}", kind),
+            SyntaxErrorKind::Unclosed(open) => format!("unclosed '{}'", open),
+            SyntaxErrorKind::EndOfFile => "unexpected end of file".to_string(),
+        }
+    }
+}
-/// range points at (or immediately follows) the actual offending token,
+/// One precise error per damaged region of the parse tree, in source order.
+///
+/// Tree-sitter's recovery often wraps a whole item in an `ERROR` node, so the
-/// which is more useful for a diagnostic than the enclosing node's range.
+/// node's own range is a poor position. Instead the innermost `ERROR`/MISSING
+/// node (first in source order) inside each damaged region is located:
+/// - a MISSING node is `expected <kind>`;
+/// - an `ERROR` holding a single skipped token is that token (`foo(1, 2` followed by
+///   `y := 3` blames `y`);
+/// - an `ERROR` that only wraps tokens the parser had already accepted (it
+///   gave up on them after seeing what came next) blames the first token
+///   after it, or end of file when nothing follows.
-fn firstErrorNode(node: tree_sitter::Node) -> Option<tree_sitter::Node> {
+pub fn syntaxErrors(root: tree_sitter::Node, text: &str) -> Vec<SyntaxError> {
+    let mut out: Vec<SyntaxError> = Vec::new();
+    collectSyntaxErrors(root, root, text, &mut out);
+    out.dedup_by(|a, b| a.start == b.start);
+    out
+}
+fn collectSyntaxErrors(
+    root: tree_sitter::Node,
+    node: tree_sitter::Node,
+    text: &str,
+    out: &mut Vec<SyntaxError>,
+) {
+    if !(node.has_error() || node.is_error() || node.is_missing()) {
+        return;
+    }
     if node.is_error() || node.is_missing() {
+        if let Some(inner) = innermostError(node) {
+            out.push(locateError(inner, root, text));
+        }
-        return Some(node);
+        return;
     }
     let mut cursor = node.walk();
     for child in node.children(&mut cursor) {
+        collectSyntaxErrors(root, child, text, out);
+    }
+}
+/// The first `ERROR`/MISSING node in source order that contains no other.
+fn innermostError(node: tree_sitter::Node) -> Option<tree_sitter::Node> {
+    if !(node.has_error() || node.is_error() || node.is_missing()) {
+        return None;
+    }
+    let mut cursor = node.walk();
+    for child in node.children(&mut cursor) {
-        if let Some(e) = firstErrorNode(child) {
+        if let Some(e) = innermostError(child) {
             return Some(e);
         }
     }
+    (node.is_error() || node.is_missing()).then_some(node)
+}
+fn locateError(err: tree_sitter::Node, root: tree_sitter::Node, text: &str) -> SyntaxError {
+    if err.is_missing() {
+        return SyntaxError {
+            start: err.start_position(),
+            end: err.end_position(),
+            kind: SyntaxErrorKind::Missing(err.kind().to_string()),
+        };
+    }
+    let skipped_token = err.child_count() == 0
+        || (err.child_count() == 1 && err.child(0).is_some_and(|c| c.child_count() == 0));
+    let token = if skipped_token {
+        Some(err)
+    } else {
+        leafStartingAtOrAfter(root, err.end_byte())
+    };
+    let eof = || {
+        let end = lastLeaf(root).map_or(root.end_position(), |l| l.end_position());
+        SyntaxError {
+            start: end,
+            end,
+            kind: SyntaxErrorKind::EndOfFile,
+        }
+    };
+    if skipped_token && err.child_count() == 1 {
+        let tok = err.child(0).unwrap();
+        // An opener the file never closes, wherever the parser gave up on it.
+        if let Some(closer) = closerOf(tok.kind()) {
+            if !isClosed(root, tok, closer) {
+                return SyntaxError {
+                    start: tok.start_position(),
+                    end: tok.end_position(),
+                    kind: SyntaxErrorKind::Unclosed(tok.kind().to_string()),
+                };
+            }
+        }
+        // A trailing operator or keyword is a body cut short, not a stray token.
+        if !tok.is_named()
+            && !matches!(tok.kind(), ")" | "]" | "}")
+            && leafStartingAtOrAfter(root, err.end_byte()).is_none()
+        {
+            return eof();
+        }
+    }
+    let Some(token) = token else {
+        return eof();
+    };
+    let (mut end_byte, mut end) = (token.end_byte(), token.end_position());
+    // An empty `()` / `[]` / `{}` reads as one token.
+    if let Some(closer) = closerOf(token.kind()) {
+        if let Some(next) = leafStartingAtOrAfter(root, token.end_byte()) {
+            if next.kind() == closer && next.start_byte() == token.end_byte() {
+                end_byte = next.end_byte();
+                end = next.end_position();
+            }
+        }
+    }
+    let raw = text.get(token.start_byte()..end_byte).unwrap_or("");
+    let line = raw.lines().next().unwrap_or("").trim();
+    let shown: String = if line.chars().count() > 40 {
+        line.chars().take(40).collect::<String>() + "..."
+    } else {
+        line.to_string()
+    };
+    SyntaxError {
+        start: token.start_position(),
+        end,
+        kind: SyntaxErrorKind::Unexpected(shown),
+    }
+}
+fn closerOf(kind: &str) -> Option<&'static str> {
+    match kind {
+        "(" => Some(")"),
+        "[" => Some("]"),
+        "{" => Some("}"),
+        _ => None,
+    }
+}
+/// Whether `open` has a matching `closer` later in the file, counting nesting.
+fn isClosed(root: tree_sitter::Node, open: tree_sitter::Node, closer: &str) -> bool {
+    let mut depth = 1usize;
+    let mut at = open.end_byte();
+    while let Some(leaf) = leafStartingAtOrAfter(root, at) {
+        if leaf.kind() == open.kind() {
+            depth += 1;
+        } else if leaf.kind() == closer {
+            depth -= 1;
+            if depth == 0 {
+                return true;
+            }
+        }
+        at = leaf.end_byte();
+    }
+    false
+}
+/// The first non-comment leaf whose start is at or after `byte`.
+fn leafStartingAtOrAfter(node: tree_sitter::Node, byte: usize) -> Option<tree_sitter::Node> {
+    if node.end_byte() <= byte && node.child_count() > 0 {
+        return None;
+    }
+    if node.child_count() == 0 {
+        return (node.start_byte() >= byte
+            && !node.is_extra()
+            && node.end_byte() > node.start_byte())
+        .then_some(node);
+    }
+    let mut cursor = node.walk();
+    for child in node.children(&mut cursor) {
+        if let Some(l) = leafStartingAtOrAfter(child, byte) {
+            return Some(l);
+        }
+    }
     None
 }
+fn lastLeaf(node: tree_sitter::Node) -> Option<tree_sitter::Node> {
+    let mut cursor = node.walk();
+    let children: Vec<_> = node.children(&mut cursor).collect();
+    for child in children.into_iter().rev() {
+        if let Some(l) = lastLeaf(child) {
+            return Some(l);
+        }
+    }
+    (node.child_count() == 0 && !node.is_extra() && node.end_byte() > node.start_byte())
+        .then_some(node)
+}
 thread_local! {
     static MERGED_LIT_ERRORS: std::cell::RefCell<Vec<(usize, usize, String)>> =
         std::cell::RefCell::new(Vec::new());

plum-core/tests/loader_test.rs CHANGED Viewed

@@ -267,6 +267,58 @@ fn removedSyntaxIsAParseError() {
     }
 }
+fn parseErrorOf(src: &str) -> String {
+    let dir = TempDir::new().unwrap();
+    let entry = dir.path().join("main.plum");
+    fs::write(&entry, src).unwrap();
+    loadAndMerge(&entry, dir.path()).unwrap_err()
+}
+#[test]
+fn parseErrorBlamesTheTokenNotTheWholeFunction() {
+    let err = parseErrorOf(
+        "import std/Float\n\nexport fun update(w: Float, dt: Float) =\n  ()\n\nfun ok() =\n  _ := 1.0\n",
+    );
+    assert!(
+        err.contains("at line 4, column 3: unexpected '()'"),
+        "{err}"
+    );
+    assert!(err.contains("hint:") && err.contains("_ := 0"), "{err}");
+    assert!(!err.contains("export fun"), "{err}");
+}
+#[test]
+fn parseErrorOnStrayTokenMidExpression() {
+    let err = parseErrorOf("fun f() =\n  x := 1 + * 2\n");
+    assert!(
+        err.contains("at line 2, column 12: unexpected '*'"),
+        "{err}"
+    );
+    assert!(!err.contains("hint:"), "{err}");
+}
+#[test]
+fn parseErrorOnMissingClosingDelimiter() {
+    let err = parseErrorOf("fun f() =\n  x := (1 + 2\n");
+    assert!(err.contains("at line 2, column 8: unclosed '('"), "{err}");
+    let err = parseErrorOf("fun f() =\n  x := foo(1, 2\n  y := 3\n\nfun g() =\n  z := 1\n");
+    assert!(err.contains("at line 3, column 3: unexpected 'y'"), "{err}");
+}
+#[test]
+fn parseErrorAtEndOfFile() {
+    let err = parseErrorOf("fun f() =\n  x := 1 +\n");
+    assert!(
+        err.contains("at line 2, column 11: unexpected end of file"),
+        "{err}"
+    );
+    let err = parseErrorOf("fun f() =\n  x := foo(1, 2");
+    assert!(
+        err.contains("at line 2, column 16: unexpected end of file"),
+        "{err}"
+    );
+}
 #[test]
 fn duplicate_const_in_one_body() {
     let dir = TempDir::new().unwrap();

plum-lsp/src/main.rs CHANGED Viewed

@@ -53,46 +53,30 @@ impl Backend {
             .expect("tree-sitter parse returned None")
     }
-    /// Real-span diagnostics straight from the parse tree: any ERROR or
+    /// Syntax diagnostics from the parse tree, one per damaged region, each
+    /// on the token tree-sitter could not fit (`plum_core::syntaxErrors`),
-    /// MISSING node becomes one diagnostic at its own range.
+    /// not on the item its recovery wrapped in an ERROR node.
-    fn syntaxDiagnostics(tree: &tree_sitter::Tree) -> Vec<Diagnostic> {
+    fn syntaxDiagnostics(tree: &tree_sitter::Tree, source: &str) -> Vec<Diagnostic> {
+        plum_core::syntaxErrors(tree.root_node(), source)
+            .into_iter()
+            .map(|e| {
-        let mut out = Vec::new();
+                let mut start = e.start;
-        let mut cursor = tree.walk();
-        let mut visit = |node: tree_sitter::Node| {
+                // A zero-width range (end of file, a missing node) underlines nothing.
-            if node.is_error() || node.is_missing() {
+                if e.end == e.start && start.column > 0 {
-                let start = node.start_position();
+                    start.column -= 1;
-                let end = node.end_position();
+                }
-                out.push(Diagnostic {
+                Diagnostic {
                     range: Range {
                         start: Position::new(start.row as u32, start.column as u32),
-                        end: Position::new(end.row as u32, end.column as u32),
+                        end: Position::new(e.end.row as u32, e.end.column as u32),
                     },
                     severity: Some(DiagnosticSeverity::ERROR),
                     source: Some("plum-syntax".into()),
-                    message: if node.is_missing() {
+                    message: e.message(),
-                        format!("missing {}", node.kind())
-                    } else {
-                        "syntax error".into()
-                    },
                     ..Default::default()
-                });
-            }
-        };
-        // Manual depth-first walk since `TreeCursor` has no built-in visitor.
-        loop {
-            visit(cursor.node());
-            if cursor.goto_first_child() {
-                continue;
-            }
-            loop {
-                if cursor.goto_next_sibling() {
-                    break;
                 }
-                if !cursor.goto_parent() {
-                    return out;
-                }
+            })
-            }
+            .collect()
-        }
     }
     /// Diagnostics from `plum_checker::checkSource`, run against the live
@@ -191,7 +175,7 @@ impl Backend {
             None => return,
         };
         let tree = Self::parseTree(&text);
-        let mut diagnostics = Self::syntaxDiagnostics(&tree);
+        let mut diagnostics = Self::syntaxDiagnostics(&tree, &text);
         if diagnostics.is_empty() {
             if let Some(mut typeErrors) = self.typeDiagnostics(&uri, &text) {
                 diagnostics.append(&mut typeErrors);
@@ -389,7 +373,7 @@ mod tests {
         let path = repoRoot().join("plum-examples/seq.plum");
         let text = std::fs::read_to_string(&path).unwrap();
         let tree = Backend::parseTree(&text);
-        assert!(Backend::syntaxDiagnostics(&tree).is_empty());
+        assert!(Backend::syntaxDiagnostics(&tree, &text).is_empty());
         let diags = typeDiagnosticsFor(&path, &repoRoot(), &text).expect("seq.plum loads");
         assert!(
             diags.is_empty(),
@@ -412,7 +396,10 @@ mod tests {
             let path = repoRoot().join("plum-examples").join(f);
             let text = std::fs::read_to_string(&path).unwrap();
             let tree = Backend::parseTree(&text);
+            assert!(
-            assert!(Backend::syntaxDiagnostics(&tree).is_empty(), "{f}: syntax");
+                Backend::syntaxDiagnostics(&tree, &text).is_empty(),
+                "{f}: syntax"
+            );
             let diags = typeDiagnosticsFor(&path, &repoRoot(), &text).expect("loads");
             assert!(
                 diags.is_empty(),
@@ -630,7 +617,7 @@ mod tests {
             let text = std::fs::read_to_string(&path).unwrap();
             let tree = Backend::parseTree(&text);
             assert!(
-                Backend::syntaxDiagnostics(&tree).is_empty(),
+                Backend::syntaxDiagnostics(&tree, &text).is_empty(),
                 "{path:?}: syntax"
             );
             let diags = typeDiagnosticsFor(&path, &repoRoot(), &text).expect("loads");
@@ -691,12 +678,12 @@ mod tests {
                 s
             };
             let tree = Backend::parseTree(&src);
-            let d = Backend::syntaxDiagnostics(&tree);
+            let d = Backend::syntaxDiagnostics(&tree, &src);
             // The one mixed block (items + statements) is parsed as two halves below.
             if block.contains("\nintNum") {
                 let (items, stmts) = block.split_at(block.find("\nintNum").unwrap() + 1);
                 assert!(
-                    Backend::syntaxDiagnostics(&Backend::parseTree(items)).is_empty(),
+                    Backend::syntaxDiagnostics(&Backend::parseTree(items), items).is_empty(),
                     "items:\n{items}"
                 );
                 let wrapped = format!(
@@ -707,7 +694,7 @@ mod tests {
                         .collect::<String>()
                 );
                 assert!(
-                    Backend::syntaxDiagnostics(&Backend::parseTree(&wrapped)).is_empty(),
+                    Backend::syntaxDiagnostics(&Backend::parseTree(&wrapped), &wrapped).is_empty(),
                     "stmts:\n{stmts}"
                 );
             } else {
@@ -717,4 +704,18 @@ mod tests {
         }
         assert!(n > 20, "found only {n} README snippets");
     }
+    #[test]
+    fn syntaxDiagnosticUnderlinesTheTokenNotTheFunction() {
+        let src = "import std/Float\n\nexport fun update(w: Float, dt: Float) =\n  ()\n\nfun ok() =\n  _ := 1.0\n";
+        let d = Backend::syntaxDiagnostics(&Backend::parseTree(src), src);
+        assert_eq!(d.len(), 1, "{d:?}");
+        assert_eq!(d[0].range.start, Position::new(3, 2));
+        assert_eq!(d[0].range.end, Position::new(3, 4));
+        assert!(
+            d[0].message.starts_with("unexpected '()'"),
+            "{}",
+            d[0].message
+        );
+    }
 }