Evaluation API Reference

Quality Estimators

vietquill.evaluation.estimators.neural_est.EnsembleModelForParaphraseQualityEstimation(hub_id: str = None, device: str = None, **kwargs)

Bases: BaseEstimator

Ensemble Quality Estimator that uses two specialized Sequence Classification models (Regression) for declarative sentences and questions, loaded from subfolders ('sentence' and 'question') of a unified Hugging Face repository.

Initializes the Ensemble Quality Estimation Model.

Parameters:

Name	Type	Description	Default
hub_id	str	Path or HF name for the unified model repo.	None
device	str	Device to run the models on ('cuda' or 'cpu').	None
**kwargs		Additional keyword arguments passed to AutoModelForSequenceClassification.from_pretrained.	{}

Source code in src/vietquill/evaluation/estimators/neural_est.py

def __init__(self, hub_id: str = None, device: str = None, **kwargs):
    """
    Initializes the Ensemble Quality Estimation Model.

    Args:
        hub_id: Path or HF name for the unified model repo.
        device: Device to run the models on ('cuda' or 'cpu').
        **kwargs: Additional keyword arguments passed to AutoModelForSequenceClassification.from_pretrained.
    """
    default_hub_id = get_config(
        "models.estimators.hub_id", MODELS["estimators"]["hub_id"]
    )
    self.hub_id = hub_id or default_hub_id
    self.device = (
        device if device else ("cuda" if torch.cuda.is_available() else "cpu")
    )

    super().__init__(**kwargs)

    logger.info(f"Loading tokenizer from {self.hub_id}...")
    self.tokenizer = AutoTokenizer.from_pretrained(self.hub_id)

    logger.info(f"Loading sentence estimator from {self.hub_id}/sentence...")
    self.model_sentence = AutoModelForSequenceClassification.from_pretrained(
        self.hub_id, subfolder="sentence", **kwargs
    )
    self.model_sentence.to(self.device)
    self.model_sentence.eval()

    logger.info(f"Loading question estimator from {self.hub_id}/question...")
    self.model_question = AutoModelForSequenceClassification.from_pretrained(
        self.hub_id, subfolder="question", **kwargs
    )
    self.model_question.to(self.device)
    self.model_question.eval()

estimate(sentence1: str, sentence2: str) -> Dict[str, float]

Estimate lexical, syntactic, and semantic scores using the appropriate trained model. Automatically detects if sentence1 is a question.

Parameters:

Name	Type	Description	Default
sentence1	str	The source sentence.	required
sentence2	str	The target (paraphrase) sentence.	required

Returns:

Type	Description
Dict[str, float]	Dictionary containing predicted scores (lexical_score, syntactic_score, semantic_score).

Source code in src/vietquill/evaluation/estimators/neural_est.py

def estimate(self, sentence1: str, sentence2: str) -> Dict[str, float]:
    """
    Estimate lexical, syntactic, and semantic scores using the appropriate trained model.
    Automatically detects if sentence1 is a question.

    Args:
        sentence1: The source sentence.
        sentence2: The target (paraphrase) sentence.

    Returns:
        Dictionary containing predicted scores (lexical_score, syntactic_score, semantic_score).
    """
    is_question = sentence1.strip().endswith("?")
    model = self.model_question if is_question else self.model_sentence

    input_text = f"{sentence1} [SEP] {sentence2}"

    inputs = self.tokenizer(
        input_text,
        return_tensors="pt",
        padding=True,
        truncation=True,
        max_length=256,
    ).to(self.device)

    with torch.no_grad():
        outputs = model(**inputs)
        predictions = outputs.logits.cpu().numpy()[0]

    # Clip scores to [0, 100] range
    lexical_score = max(0.0, min(100.0, float(predictions[0])))
    syntactic_score = max(0.0, min(100.0, float(predictions[1])))
    semantic_score = max(0.0, min(100.0, float(predictions[2])))

    return {
        "lexical_score": round(lexical_score, 2),
        "syntactic_score": round(syntactic_score, 2),
        "semantic_score": round(semantic_score, 2),
    }

vietquill.evaluation.estimators.neural_est.AutoModelForQualityEstimation(hub_id: str = None, device: str = None, **kwargs)

Bases: BaseEstimator

Quality Estimator using a single trained Sequence Classification model (Regression) directly loaded from Hugging Face Hub or a local path, without sentence/question splitting.

Initializes the Quality Estimation Model with a single specified model.

Parameters:

Name	Type	Description	Default
hub_id	str	Path or HF repo ID for the model repository or checkpoint.	None
device	str	Device to run the model on ('cuda' or 'cpu').	None
**kwargs		Additional keyword arguments passed to AutoModelForSequenceClassification.from_pretrained.	{}

Source code in src/vietquill/evaluation/estimators/neural_est.py

def __init__(self, hub_id: str = None, device: str = None, **kwargs):
    """
    Initializes the Quality Estimation Model with a single specified model.

    Args:
        hub_id: Path or HF repo ID for the model repository or checkpoint.
        device: Device to run the model on ('cuda' or 'cpu').
        **kwargs: Additional keyword arguments passed to AutoModelForSequenceClassification.from_pretrained.
    """
    default_hub_id = get_config(
        "models.estimators.hub_id", MODELS["estimators"]["hub_id"]
    )
    self.hub_id = hub_id or default_hub_id
    self.device = (
        device if device else ("cuda" if torch.cuda.is_available() else "cpu")
    )

    super().__init__(**kwargs)

    logger.info(f"Loading tokenizer from {self.hub_id}...")
    self.tokenizer = AutoTokenizer.from_pretrained(self.hub_id)

    logger.info(f"Loading model from {self.hub_id}...")
    self.model = AutoModelForSequenceClassification.from_pretrained(
        self.hub_id, **kwargs
    )
    self.model.to(self.device)
    self.model.eval()

estimate(sentence1: str, sentence2: str) -> Dict[str, float]

Estimate lexical, syntactic, and semantic scores using the loaded model.

Parameters:

Name	Type	Description	Default
sentence1	str	The source sentence.	required
sentence2	str	The target (paraphrase) sentence.	required

Returns:

Type	Description
Dict[str, float]	Dictionary containing predicted scores (lexical_score, syntactic_score, semantic_score).

Source code in src/vietquill/evaluation/estimators/neural_est.py

def estimate(self, sentence1: str, sentence2: str) -> Dict[str, float]:
    """
    Estimate lexical, syntactic, and semantic scores using the loaded model.

    Args:
        sentence1: The source sentence.
        sentence2: The target (paraphrase) sentence.

    Returns:
        Dictionary containing predicted scores (lexical_score, syntactic_score, semantic_score).
    """
    input_text = f"{sentence1} [SEP] {sentence2}"

    inputs = self.tokenizer(
        input_text,
        return_tensors="pt",
        padding=True,
        truncation=True,
        max_length=256,
    ).to(self.device)

    with torch.no_grad():
        outputs = self.model(**inputs)
        predictions = outputs.logits.cpu().numpy()[0]

    # Clip scores to [0, 100] range
    lexical_score = max(0.0, min(100.0, float(predictions[0])))
    syntactic_score = max(0.0, min(100.0, float(predictions[1])))
    semantic_score = max(0.0, min(100.0, float(predictions[2])))

    return {
        "lexical_score": round(lexical_score, 2),
        "syntactic_score": round(syntactic_score, 2),
        "semantic_score": round(semantic_score, 2),
    }

vietquill.evaluation.estimators.lexical_est.LexicalEstimator(model=None, tokenizer='whitespace', **kwargs)

Bases: BaseEstimator

Estimator for lexical similarity between Vietnamese sentences using Jaccard similarity.

Source code in src/vietquill/evaluation/estimators/lexical_est.py

def __init__(self, model=None, tokenizer="whitespace", **kwargs):
    super().__init__(model=model, **kwargs)
    self.lexical_metric = JaccardMetric(tokenizer=tokenizer)

estimate(sentence1, sentence2)

Estimate the lexical structure similarity of the given sentences.

Parameters:

Name	Type	Description	Default
sentence1	str	The first sentence to compare.	required
sentence2	str	The second sentence to compare.	required

Returns:

Name	Type	Description
dict		A dictionary containing the estimated lexical score.

Source code in src/vietquill/evaluation/estimators/lexical_est.py

def estimate(self, sentence1, sentence2):
    """
    Estimate the lexical structure similarity of the given sentences.

    Args:
        sentence1 (str): The first sentence to compare.
        sentence2 (str): The second sentence to compare.

    Returns:
        dict: A dictionary containing the estimated lexical score.
    """
    lexical_score = self.lexical_metric.score(sentence1, sentence2)
    return {"lexical_score": round(lexical_score * 100, 2)}

vietquill.evaluation.estimators.syntactic_est.SyntacticEstimator(model=None, max_depth=3, **kwargs)

Bases: BaseEstimator

Estimator for syntactic structure similarity between Vietnamese sentences using TED (Tree Edit Distance).

Source code in src/vietquill/evaluation/estimators/syntactic_est.py

def __init__(self, model=None, max_depth=3, **kwargs):
    super().__init__(model=model, **kwargs)
    self.syntactic_metric = TEDMetric(max_depth=max_depth)

estimate(sentence1, sentence2)

Estimate the syntactic structure similarity of the given sentences.

Parameters:

Name	Type	Description	Default
sentence1	str	The first sentence to compare.	required
sentence2	str	The second sentence to compare.	required

Returns:

Name	Type	Description
dict		A dictionary containing the estimated syntactic score.

Source code in src/vietquill/evaluation/estimators/syntactic_est.py

def estimate(self, sentence1, sentence2):
    """
    Estimate the syntactic structure similarity of the given sentences.

    Args:
        sentence1 (str): The first sentence to compare.
        sentence2 (str): The second sentence to compare.

    Returns:
        dict: A dictionary containing the estimated syntactic score.
    """
    syntactic_score = self.syntactic_metric.score(sentence1, sentence2)
    return {"syntactic_score": round(syntactic_score * 100, 2)}

vietquill.evaluation.estimators.semantic_est.SemanticEstimator(model_type=None, device=None, num_layers=None, **kwargs)

Bases: BaseEstimator

Estimator for semantic similarity between Vietnamese sentences using BERTScore.

Source code in src/vietquill/evaluation/estimators/semantic_est.py

def __init__(self, model_type=None, device=None, num_layers=None, **kwargs):
    self.model_type = model_type or get_config("evaluation.bertscore.model", EVALUATION["bertscore"]["model"])
    self.num_layers = num_layers or get_config("evaluation.bertscore.num_layers", EVALUATION["bertscore"]["num_layers"])

    super().__init__(model=self.model_type, **kwargs)
    self.bertscore_metric = BERTScoreMetric(model_type=self.model_type, device=device, num_layers=self.num_layers)

estimate(sentence1, sentence2)

Estimate the semantic similarity score between two Vietnamese sentences.

The returned score is the F1 score from BERTScore scaled to [0, 100].

Parameters:

Name	Type	Description	Default
sentence1	str	The first Vietnamese sentence.	required
sentence2	str	The second Vietnamese sentence.	required

Returns:

Name	Type	Description
dict		A dictionary containing the semantic score.

Source code in src/vietquill/evaluation/estimators/semantic_est.py

def estimate(self, sentence1, sentence2):
    """
    Estimate the semantic similarity score between two Vietnamese sentences.

    The returned score is the F1 score from BERTScore scaled to [0, 100].

    Args:
        sentence1 (str): The first Vietnamese sentence.
        sentence2 (str): The second Vietnamese sentence.

    Returns:
        dict: A dictionary containing the semantic score.
    """
    f1_score = self.bertscore_metric.score(sentence1, sentence2)
    semantic_score = round(f1_score * 100, 2)

    return {
        "semantic_score": semantic_score,
    }

---

Evaluation Metrics

vietquill.evaluation.metrics.bleu_metric.BLEUMetric(tokenizer='whitespace', lowercase=True)

Bases: BaseMetric

BLEU metric for Vietnamese paraphrase evaluation.

Source code in src/vietquill/evaluation/metrics/bleu_metric.py

def __init__(self, tokenizer="whitespace", lowercase=True):
    if sentence_bleu is None:
        raise ImportError(
            "nltk is not installed. "
            "Run: pip install nltk"
        )

    self.tokenizer_name = tokenizer
    self.lowercase = lowercase
    self.scores = []
    self.smoother = SmoothingFunction().method1

compute()

Compute the average BLEU score.

Source code in src/vietquill/evaluation/metrics/bleu_metric.py

def compute(self):
    """
    Compute the average BLEU score.
    """
    if not self.scores:
        return 0.0
    return sum(self.scores) / len(self.scores)

reset()

Reset the metric state.

Source code in src/vietquill/evaluation/metrics/bleu_metric.py

def reset(self):
    """
    Reset the metric state.
    """
    self.scores = []

score(y_true, y_pred)

Compute BLEU-4 for a single sentence pair.

Source code in src/vietquill/evaluation/metrics/bleu_metric.py

def score(self, y_true, y_pred):
    """
    Compute BLEU-4 for a single sentence pair.
    """
    reference = [self.tokenize(y_true)]
    candidate = self.tokenize(y_pred)

    # BLEU-4 uses equal weights for 1-gram, 2-gram, 3-gram, and 4-gram
    weights = (0.25, 0.25, 0.25, 0.25)

    return sentence_bleu(reference, candidate, weights=weights, smoothing_function=self.smoother)

update(y_true, y_pred)

Update the running metric state.

Source code in src/vietquill/evaluation/metrics/bleu_metric.py

def update(self, y_true, y_pred):
    """
    Update the running metric state.
    """
    score_val = self.score(y_true, y_pred)
    self.scores.append(score_val)

vietquill.evaluation.metrics.bertscore_metric.BERTScoreMetric(model_type=None, lang=None, device=None, num_layers=None)

Bases: BaseMetric

BERTScore metric for semantic similarity using PhoBERT.

Source code in src/vietquill/evaluation/metrics/bertscore_metric.py

def __init__(self, model_type=None, lang=None, device=None, num_layers=None):
    if bert_score_func is None:
        raise ImportError(
            "bert_score is not installed. "
            "Run: pip install bert-score"
        )

    self.model_type = model_type or get_config("evaluation.bertscore.model", EVALUATION["bertscore"]["model"])
    self.lang = lang or get_config("evaluation.bertscore.lang", EVALUATION["bertscore"]["lang"])
    self.device = device or ('cuda' if torch.cuda.is_available() else 'cpu')
    self.num_layers = num_layers or get_config("evaluation.bertscore.num_layers", EVALUATION["bertscore"]["num_layers"])

    self.f1_scores = []

compute()

Compute the average F1 score.

Source code in src/vietquill/evaluation/metrics/bertscore_metric.py

def compute(self):
    """
    Compute the average F1 score.
    """
    if not self.f1_scores:
        return 0.0
    return sum(self.f1_scores) / len(self.f1_scores)

reset()

Reset the metric state.

Source code in src/vietquill/evaluation/metrics/bertscore_metric.py

def reset(self):
    """
    Reset the metric state.
    """
    self.f1_scores = []

score(y_true, y_pred)

Compute BERTScore for a single sentence pair.

Source code in src/vietquill/evaluation/metrics/bertscore_metric.py

def score(self, y_true, y_pred):
    """
    Compute BERTScore for a single sentence pair.
    """
    P, R, F1 = bert_score_func(
        [y_pred], [y_true],
        model_type=self.model_type,
        lang=self.lang,
        verbose=False,
        num_layers=self.num_layers,
        device=self.device
    )
    return F1[0].item()

update(y_true, y_pred)

Update the running metric state.

Source code in src/vietquill/evaluation/metrics/bertscore_metric.py

def update(self, y_true, y_pred):
    """
    Update the running metric state.
    """
    score = self.score(y_true, y_pred)
    self.f1_scores.append(score)

vietquill.evaluation.metrics.parascore_metric.ParaScoreMetric(lang=None, model_type=None, batch_size=None, num_layers=None)

Bases: BaseMetric

ParaScore metric for paraphrase evaluation.

Supports

• ParaScore (reference-based)
• ParaScore.Free (reference-free)

Source code in src/vietquill/evaluation/metrics/parascore_metric.py

def __init__(
    self,
    lang=None,
    model_type=None,
    batch_size=None,
    num_layers=None
):
    if ParaScorer is None:
        raise ImportError(
            "parascore is not installed. "
            "Run: pip install parascore"
        )

    self.lang = lang or get_config("evaluation.parascore.lang", EVALUATION["parascore"]["lang"])
    self.model_type = model_type or get_config("evaluation.parascore.model", EVALUATION["parascore"]["model"])
    self.num_layers = num_layers or get_config("evaluation.parascore.num_layers", EVALUATION["parascore"]["num_layers"])
    self.batch_size = batch_size or get_config("evaluation.parascore.batch_size", EVALUATION["parascore"]["batch_size"])

    self.scorer = ParaScorer(
        lang=self.lang,
        model_type=self.model_type,
        num_layers=self.num_layers
    )

    self.parascore_scores = []
    self.parascore_free_scores = []

compute()

Return average scores.

Source code in src/vietquill/evaluation/metrics/parascore_metric.py

def compute(self):
    """
    Return average scores.
    """

    avg_para = (
        sum(self.parascore_scores)
        / len(self.parascore_scores)
        if self.parascore_scores
        else 0.0
    )

    avg_free = (
        sum(self.parascore_free_scores)
        / len(self.parascore_free_scores)
        if self.parascore_free_scores
        else 0.0
    )

    return {
        "ParaScore": avg_para,
        "ParaScore.Free": avg_free
    }

score(source, candidate, reference)

Compute ParaScore.

Source code in src/vietquill/evaluation/metrics/parascore_metric.py

def score(self, source, candidate, reference):
    """
    Compute ParaScore.
    """

    score = self.scorer.base_score(
        [candidate],
        [source],
        [reference],
        batch_size=self.batch_size
    )

    return float(score[0])

score_free(source, candidate)

Compute ParaScore.Free.

Source code in src/vietquill/evaluation/metrics/parascore_metric.py

def score_free(self, source, candidate):
    """
    Compute ParaScore.Free.
    """

    score = self.scorer.free_score(
        [candidate],
        [source],
        batch_size=self.batch_size
    )

    return float(score[0])

update(source, candidate, reference)

Update running state.

Source code in src/vietquill/evaluation/metrics/parascore_metric.py

def update(self, source, candidate, reference):
    """
    Update running state.
    """

    para_score = self.score(
        source,
        candidate,
        reference
    )

    para_free_score = self.score_free(
        source,
        candidate
    )

    self.parascore_scores.append(para_score)
    self.parascore_free_scores.append(
        para_free_score
    )

vietquill.evaluation.metrics.ted_metric.TEDMetric(max_depth: int = 3)

Bases: BaseMetric

Compute a normalized Tree Edit Distance (TED) similarity over bracket trees.

Source code in src/vietquill/evaluation/metrics/ted_metric.py

def __init__(self, max_depth: int = 3):
    self.max_depth = max_depth
    self.total_sim = 0.0
    self.count = 0

normalize_tree(tree_string: str) -> str

Convert a standard bracket tree to a '{' '}' style with depth truncation.

Source code in src/vietquill/evaluation/metrics/ted_metric.py

def normalize_tree(self, tree_string: str) -> str:
    """
    Convert a standard bracket tree to a '{' '}' style with depth truncation.
    """
    res = []
    depth = -1
    leaf = False
    for c in tree_string:
        if c in ['{', '}']:
            continue
        if c == '(':
            leaf = False
            depth += 1
        elif c == ')':
            leaf = False
            depth -= 1
            if depth < self.max_depth:
                res.append('}')
                continue
        elif c == ' ':
            leaf = True
            continue
        if depth <= self.max_depth and not leaf and c != ')':
            res.append(c if c != '(' else '{')
    return ''.join(res)

score(sentence1, sentence2)

Compute syntactic similarity score in [0, 1].

Source code in src/vietquill/evaluation/metrics/ted_metric.py

def score(self, sentence1, sentence2):
    """
    Compute syntactic similarity score in [0, 1].
    """
    s1 = (sentence1 or "").strip()
    s2 = (sentence2 or "").strip()
    if not s1 and not s2:
        return 1.0
    if not s1 or not s2:
        return 0.0

    nlp = _init_vi_pipeline()
    doc1 = nlp(s1)
    doc2 = nlp(s2)

    if not doc1.sentences or not doc2.sentences:
        return 0.0

    try:
        tree1 = str(doc1.sentences[0].constituency)
        tree2 = str(doc2.sentences[0].constituency)
    except Exception:
        return 0.0

    t1n = self.normalize_tree(tree1)
    t2n = self.normalize_tree(tree2)
    ted_norm = self.tree_edit_distance(t1n, t2n)
    sim = max(0.0, min(1.0, 1.0 - ted_norm))
    return sim

tree_edit_distance(lintree1: str, lintree2: str) -> float

Return normalized TED: raw_edit_distance / (nodes1 + nodes2).

Source code in src/vietquill/evaluation/metrics/ted_metric.py

def tree_edit_distance(self, lintree1: str, lintree2: str) -> float:
    """
    Return normalized TED: raw_edit_distance / (nodes1 + nodes2).
    """
    try:
        t1 = Tree.from_text(lintree1)
        t2 = Tree.from_text(lintree2)
    except Exception:
        return 1.0

    n1 = lintree1.count('{')
    n2 = lintree2.count('{')
    if (n1 + n2) == 0:
        return 0.0

    ted = APTED(t1, t2).compute_edit_distance()
    return ted / (n1 + n2)

vietquill.evaluation.metrics.jaccard_metric.JaccardMetric(tokenizer='whitespace', lowercase=True)

Bases: BaseMetric

Jaccard similarity for sentence overlap.

Supported tokenizers

• whitespace
• underthesea
• pyvi

Source code in src/vietquill/evaluation/metrics/jaccard_metric.py

def __init__(self, tokenizer="whitespace", lowercase=True):
    self.intersection = 0
    self.union = 0
    self.lowercase = lowercase
    self.tokenizer_name = tokenizer

score(y_true, y_pred)

Compute Jaccard for a single sentence pair.

Source code in src/vietquill/evaluation/metrics/jaccard_metric.py

def score(self, y_true, y_pred):
    """
    Compute Jaccard for a single sentence pair.
    """
    set_true = set(self.tokenize(y_true))
    set_pred = set(self.tokenize(y_pred))

    if len(set_true | set_pred) == 0:
        return 1.0

    return len(set_true & set_pred) / len(set_true | set_pred)

update(y_true, y_pred)

Input

y_true: str y_pred: str

Source code in src/vietquill/evaluation/metrics/jaccard_metric.py

def update(self, y_true, y_pred):
    """
    Input:
        y_true: str
        y_pred: str
    """
    set_true = set(self.tokenize(y_true))
    set_pred = set(self.tokenize(y_pred))

    self.intersection += len(set_true & set_pred)
    self.union += len(set_true | set_pred)