001/*
002 * Copyright (c) 2015-2020, Oracle and/or its affiliates. All rights reserved.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016
017package org.tribuo.sequence;
018
019import org.tribuo.Output;
020import org.tribuo.Prediction;
021import org.tribuo.provenance.DataProvenance;
022
023import java.util.List;
024
025/**
026 * An evaluation factory which produces immutable {@link SequenceEvaluation}s of a given {@link SequenceDataset} using the given {@link SequenceModel}.
027 * @param <T> The output type.
028 * @param <E> The evaluation type.
029 */
030public interface SequenceEvaluator<T extends Output<T>, E extends SequenceEvaluation<T>> {
031
032    /**
033     * Evaluates the dataset using the supplied model, returning an immutable evaluation.
034     * @param model The model to use.
035     * @param dataset The dataset to evaluate.
036     * @return An evaluation.
037     */
038    public E evaluate(SequenceModel<T> model, SequenceDataset<T> dataset);
039
040    /**
041     * Evaluates the datasource using the supplied model, returning an immutable evaluation.
042     * @param model The model to use.
043     * @param datasource The datasource to evaluate.
044     * @return An evaluation.
045     */
046    public E evaluate(SequenceModel<T> model, SequenceDataSource<T> datasource);
047
048    /**
049     * Evaluates the supplied model and predictions by aggregating the appropriate statistics.
050     * <p>
051     * Warning, this method cannot validate that the predictions were returned by the model in question.
052     * @param model The model to use.
053     * @param predictions The predictions to use.
054     * @param dataProvenance The provenance of the test data.
055     * @return An evaluation of the predictions.
056     */
057    public E evaluate(SequenceModel<T> model, List<List<Prediction<T>>> predictions, DataProvenance dataProvenance);
058
059}