001/*
002 * Copyright (c) 2015-2020, Oracle and/or its affiliates. All rights reserved.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016
017package org.tribuo.classification.evaluation;
018
019import org.tribuo.classification.Classifiable;
020import org.tribuo.evaluation.Evaluation;
021
022/**
023 * Defines methods that calculate classification performance, used for both multi-class and multi-label classification.
024 *
025 * @param <T> The output type.
026 */
027public interface ClassifierEvaluation<T extends Classifiable<T>> extends Evaluation<T> {
028
029    /**
030     * Returns the number of times label {@code truth} was predicted as label {@code predicted}.
031     *
032     * @param predicted The predicted label.
033     * @param truth     The true label.
034     * @return The number of times the predicted label was returned for the true label.
035     */
036    public double confusion(T predicted, T truth);
037
038    // TODO add support(), support(label)
039
040    /**
041     * Returns the number of true positives, i.e., the number of times the label was correctly predicted.
042     *
043     * @param label The label to calculate.
044     * @return The number of true positives for that label.
045     */
046    public double tp(T label);
047
048    /**
049     * Returns the micro average of the number of true positives across all the labels, i.e., the total
050     * number of true positives.
051     *
052     * @return The micro averaged number of true positives.
053     */
054    public double tp();
055
056    /**
057     * Returns the macro averaged number of true positives, averaged across the labels.
058     *
059     * @return The macro averaged number of true positives.
060     */
061    public double macroTP();
062
063    /**
064     * Returns the number of false positives, i.e., the number of times this label was predicted but it was not the true label..
065     *
066     * @param label the label to calculate.
067     * @return The number of false positives for that label.
068     */
069    public double fp(T label);
070
071    /**
072     * Returns the micro average of the number of false positives across all the labels, i.e., the total
073     * number of false positives.
074     *
075     * @return The micro averaged number of false positives.
076     */
077    public double fp();
078
079    /**
080     * Returns the macro averaged number of false positives, averaged across the labels.
081     *
082     * @return The macro averaged number of false positives.
083     */
084    public double macroFP();
085
086    /**
087     * Returns the number of true negatives for that label, i.e., the number of times it wasn't predicted, and was not the true label.
088     *
089     * @param label The label to use.
090     * @return the number of true negatives.
091     */
092    public double tn(T label);
093
094    /**
095     * Returns the total number of true negatives. This isn't very useful in multiclass problems.
096     *
097     * @return The number of true negatives.
098     */
099    public double tn();
100
101    /**
102     * Returns the macro averaged number of true negatives.
103     *
104     * @return The macro averaged number of true negatives.
105     */
106    public double macroTN();
107
108    /**
109     * Returns the number of false negatives, i.e., the number of times the true label was incorrectly predicted as another label.
110     *
111     * @param label The true label.
112     * @return The number of false negatives.
113     */
114    public double fn(T label);
115
116    /**
117     * Returns the micro averaged number of false negatives.
118     *
119     * @return The micro averaged number of false negatives.
120     */
121    public double fn();
122
123    /**
124     * Returns the macro averaged number of false negatives.
125     *
126     * @return The macro averaged number of false negatives.
127     */
128    public double macroFN();
129
130    /**
131     * Returns the precision of this label, i.e., the number of true positives divided by the number of true positives plus false positives.
132     *
133     * @param label The label.
134     * @return The precision.
135     */
136    public double precision(T label);
137
138    /**
139     * Returns the micro averaged precision.
140     *
141     * @return The micro averaged precision.
142     */
143    public double microAveragedPrecision();
144
145    /**
146     * Returns the macro averaged precision.
147     *
148     * @return The macro averaged precision.
149     */
150    public double macroAveragedPrecision();
151
152    /**
153     * Returns the recall of this label, i.e., the number of true positives divided by the number of true positives plus false negatives.
154     *
155     * @param label The label.
156     * @return The recall.
157     */
158    public double recall(T label);
159
160    /**
161     * Returns the micro averaged recall.
162     *
163     * @return The micro averaged recall.
164     */
165    public double microAveragedRecall();
166
167    /**
168     * Returns the macro averaged recall.
169     *
170     * @return The macro averaged recall.
171     */
172    public double macroAveragedRecall();
173
174    /**
175     * Returns the F_1 score, i.e., the harmonic mean of the precision and recall.
176     *
177     * @param label The label.
178     * @return The F_1 score.
179     */
180    public double f1(T label);
181
182    /**
183     * Returns the micro averaged F_1 across all labels.
184     *
185     * @return The F_1 score.
186     */
187    public double microAveragedF1();
188
189    /**
190     * Returns the macro averaged F_1 across all the labels.
191     *
192     * @return The F_1 score.
193     */
194    public double macroAveragedF1();
195
196    /**
197     * Returns the balanced error rate, i.e., the mean of the per label recalls.
198     *
199     * @return The balanced error rate.
200     */
201    public double balancedErrorRate();
202
203    /**
204     * Returns the underlying confusion matrix.
205     * @return The confusion matrix.
206     */
207    public ConfusionMatrix<T> getConfusionMatrix();
208}