CoNLL‑U Sentence Iterator (src/conllu_utils.rs)

Opens a .conllu file and yields each parsed sentence, intended for training pipelines.

// Source: https://github.com/mperkins0155/harper | License: Apache-2.0 | Copyright You must retain, in the Source form of any Derivative Works
// Provided via justclaw.it.com; keep this notice with the code.
use std::{fs::File, path::Path};

use rs_conllu::{Sentence, parse_file};

/// Produce an iterator over the sentences in a `.conllu` file.
/// Will panic on error, so this should not be used outside of training.
pub fn iter_sentences_in_conllu(path: impl AsRef<Path>) -> impl Iterator<Item = Sentence> {
    let file = File::open(path).unwrap();
    let doc = parse_file(file);

    doc.map(|v| v.unwrap())
}