1 package eu.fbk.dkm.pikes.naflib;
2
3 import com.google.common.base.Preconditions;
4 import com.google.common.collect.*;
5 import eu.fbk.utils.eval.ConfusionMatrix;
6 import eu.fbk.utils.eval.PrecisionRecall;
7 import eu.fbk.utils.eval.SetPrecisionRecall;
8 import eu.fbk.utils.svm.Util;
9 import ixa.kaflib.KAFDocument;
10 import ixa.kaflib.Opinion;
11 import ixa.kaflib.Opinion.OpinionExpression;
12 import ixa.kaflib.Opinion.Polarity;
13 import ixa.kaflib.Span;
14 import ixa.kaflib.Term;
15 import org.slf4j.Logger;
16 import org.slf4j.LoggerFactory;
17
18 import javax.annotation.Nullable;
19 import java.io.Serializable;
20 import java.util.*;
21 import java.util.function.BiFunction;
22 import java.util.stream.StreamSupport;
23
24
25 public final class OpinionPrecisionRecall implements Serializable {
26
27 private static final Logger LOGGER = LoggerFactory.getLogger(OpinionPrecisionRecall.class);
28
29 private static final Set<Object> NULL_TERMS = ImmutableSet.of(new Object());
30
31 private static final Set<Object> IMPLICIT = ImmutableSet.of(new Object());
32
33 private static final Set<Object> WRITER = ImmutableSet.of(new Object());
34
35 private static final long serialVersionUID = 1L;
36
37 private final ConfusionMatrix polarityCM;
38
39 private final SetPrecisionRecall[] polaritySPRsByValue;
40
41 private final SetPrecisionRecall polaritySPR;
42
43 private final SetPrecisionRecall expressionSPR;
44
45 private final SetPrecisionRecall holderSPR;
46
47 private final SetPrecisionRecall targetSPR;
48
49 public OpinionPrecisionRecall(final ConfusionMatrix polarityCM,
50 final SetPrecisionRecall[] polaritySPRsByValue, final SetPrecisionRecall polaritySPR,
51 final SetPrecisionRecall expressionSPR, final SetPrecisionRecall holderSPR,
52 final SetPrecisionRecall targetSPR) {
53
54 Preconditions.checkNotNull(polaritySPRsByValue);
55 Preconditions.checkArgument(polaritySPRsByValue.length == Polarity.values().length);
56
57 this.polarityCM = Preconditions.checkNotNull(polarityCM);
58 this.polaritySPRsByValue = polaritySPRsByValue.clone();
59 this.polaritySPR = Preconditions.checkNotNull(polaritySPR);
60 this.expressionSPR = Preconditions.checkNotNull(expressionSPR);
61 this.holderSPR = Preconditions.checkNotNull(holderSPR);
62 this.targetSPR = Preconditions.checkNotNull(targetSPR);
63 }
64
65 public ConfusionMatrix getPolarityCM() {
66 return this.polarityCM;
67 }
68
69 public SetPrecisionRecall getPolaritySPR() {
70 return this.polaritySPR;
71 }
72
73 public SetPrecisionRecall getPolaritySPR(final Polarity polarity) {
74 return Preconditions.checkNotNull(this.polaritySPRsByValue[polarity.ordinal()]);
75 }
76
77 public SetPrecisionRecall getExpressionSPR() {
78 return this.expressionSPR;
79 }
80
81 public SetPrecisionRecall getHolderSPR() {
82 return this.holderSPR;
83 }
84
85 public SetPrecisionRecall getTargetSPR() {
86 return this.targetSPR;
87 }
88
89 @Override
90 public boolean equals(final Object object) {
91 if (object == this) {
92 return true;
93 }
94 if (!(object instanceof OpinionPrecisionRecall)) {
95 return false;
96 }
97 final OpinionPrecisionRecall other = (OpinionPrecisionRecall) object;
98 return this.polarityCM.equals(other.polarityCM)
99 && this.polaritySPRsByValue.equals(other.polaritySPRsByValue)
100 && this.polaritySPR.equals(other.polaritySPR)
101 && this.expressionSPR.equals(other.expressionSPR)
102 && this.holderSPR.equals(other.holderSPR)
103 && this.targetSPR.equals(other.targetSPR);
104 }
105
106 @Override
107 public int hashCode() {
108 return Objects.hash(this.polarityCM, this.polaritySPRsByValue, this.polaritySPR,
109 this.expressionSPR, this.holderSPR, this.targetSPR);
110 }
111
112 public String toString(final boolean includePolarityCM) {
113 final StringBuilder builder = new StringBuilder();
114 builder.append(" aligned intersection"
115 + " overlap exact counts");
116 builder.append("\n p r f1 a p r f1 a "
117 + "p r f1 a p r f1 a tp fp fn");
118 builder.append("\n---------------------------------------------------------------------"
119 + "--------------------------------------------------------------------");
120 toStringHelper(builder, "expression", this.expressionSPR);
121 toStringHelper(builder, "holder", this.holderSPR);
122 toStringHelper(builder, "target", this.targetSPR);
123 toStringHelper(builder, "polarity", this.polaritySPR);
124 toStringHelper(builder, "- neutral", this.polaritySPRsByValue[Polarity.NEUTRAL.ordinal()]);
125 toStringHelper(builder, "- positive",
126 this.polaritySPRsByValue[Polarity.POSITIVE.ordinal()]);
127 toStringHelper(builder, "- negative",
128 this.polaritySPRsByValue[Polarity.NEGATIVE.ordinal()]);
129 if (includePolarityCM) {
130 builder.append("\n\n");
131 builder.append(this.polarityCM.toString("neu", "pos", "neg"));
132 }
133 return builder.toString();
134 }
135
136 @Override
137 public String toString() {
138 return toString(true);
139 }
140
141 private void toStringHelper(final StringBuilder builder, final String label,
142 final SetPrecisionRecall spr) {
143 builder.append(String.format("\n%-10s", label));
144 for (final PrecisionRecall pr : new PrecisionRecall[] { spr.getAlignedPR(),
145 spr.getIntersectionPR(), spr.getOverlapPR(), spr.getExactPR() }) {
146 builder.append(String.format(" %5.3f %5.3f %5.3f %5.3f", pr.getPrecision(),
147 pr.getRecall(), pr.getF1(), pr.getAccuracy()));
148 }
149 builder.append(String.format(" %6d %6d %6d", (int) spr.getExactPR().getTP(), (int) spr
150 .getExactPR().getFP(), (int) spr.getExactPR().getFN()));
151 }
152
153 private static Set<Object> getHolder(@Nullable final Opinion opinion) {
154 if (opinion != null) {
155 final Span<Term> holder = opinion.getHolderSpan();
156 if (holder != null && !holder.isEmpty()) {
157 return ImmutableSet.copyOf(holder.getTargets());
158 }
159 final OpinionExpression expr = opinion.getOpinionExpression();
160 if (expr != null && "implicit".equalsIgnoreCase(expr.getSentimentProductFeature())) {
161 return IMPLICIT;
162 }
163 }
164 return WRITER;
165 }
166
167 private static Set<Object> getTarget(@Nullable final Opinion opinion) {
168 if (opinion != null) {
169 final Span<Term> target = opinion.getTargetSpan();
170 if (target != null && !target.isEmpty()) {
171 return ImmutableSet.copyOf(target.getTargets());
172 }
173 }
174 return NULL_TERMS;
175 }
176
177 private static Set<Object> getExpression(@Nullable final Opinion opinion) {
178 if (opinion != null) {
179 final Span<Term> expr = opinion.getExpressionSpan();
180 if (expr != null && !expr.isEmpty()) {
181 return ImmutableSet.copyOf(expr.getTargets());
182 }
183 }
184 return NULL_TERMS;
185 }
186
187 private static Polarity getPolarity(@Nullable final Opinion opinion) {
188 return opinion == null || opinion.getOpinionExpression() == null ? Polarity.NEUTRAL
189 : Polarity.forExpression(opinion.getOpinionExpression());
190 }
191
192 private static Map<Set<Object>, Polarity> getPolarityMap(final Iterable<Opinion> opinions,
193 @Nullable final Polarity expectedPolarity) {
194 final Map<Set<Object>, Polarity> map = Maps.newHashMap();
195 for (final Opinion opinion : opinions) {
196 final Set<Object> expression = getExpression(opinion);
197 final Polarity polarity = getPolarity(opinion);
198 if (expectedPolarity != null && !expectedPolarity.equals(polarity)) {
199 continue;
200 }
201 final Polarity oldPolarity = map.put(expression, polarity);
202 if (oldPolarity != null && !Objects.equals(oldPolarity, polarity)) {
203 LOGGER.warn("Different polarities " + oldPolarity + ", " + polarity
204 + " for expression of opinion " + opinion.getId());
205 }
206 }
207 return map;
208 }
209
210 private static Multimap<Set<Object>, Opinion> indexOpinionsByExpression(
211 final Iterable<Opinion> opinions) {
212 final Multimap<Set<Object>, Opinion> map = HashMultimap.create();
213 for (final Opinion opinion : opinions) {
214 final Set<Object> expression = getExpression(opinion);
215 map.put(expression, opinion);
216 }
217 return map;
218 }
219
220 private static Multimap<Integer, Opinion> indexOpinionsBySentence(
221 final Iterable<Opinion> opinions) {
222 final ListMultimap<Integer, Opinion> map = ArrayListMultimap.create();
223 for (final Opinion opinion : opinions) {
224 final OpinionExpression oe = opinion.getOpinionExpression();
225 final Integer sent = oe == null || oe.getSpan() == null
226 || oe.getSpan().getTargets().isEmpty() ? null : oe.getSpan().getTargets()
227 .get(0).getSent();
228 map.put(sent, opinion);
229 }
230 return map;
231 }
232
233 public static BiFunction<Opinion, Opinion, List<Double>> matcher() {
234 return (final Opinion g, final Opinion t) -> {
235 final Set<Object> ge = getExpression(g);
236 final Set<Object> te = getExpression(t);
237 final Set<Object> gh = getHolder(g);
238 final Set<Object> th = getHolder(t);
239 final Set<Object> gt = getTarget(g);
240 final Set<Object> tt = getTarget(t);
241 if (Sets.intersection(ge, te).isEmpty()) {
242 return null;
243 }
244 final List<Double> scores = Lists.newArrayListWithCapacity(6);
245 scores.add(Util.coverage(ge, te));
246 scores.add(Util.coverage(te, ge));
247 scores.add(Util.coverage(gt, tt));
248 scores.add(Util.coverage(tt, gt));
249 scores.add(Util.coverage(gh, th));
250 scores.add(Util.coverage(th, gh));
251 return scores;
252 };
253 }
254
255 public static Evaluator evaluator() {
256 return new Evaluator();
257 }
258
259 public static final class Evaluator {
260
261 private final ConfusionMatrix.Evaluator polarityCMEvaluator;
262
263 private final SetPrecisionRecall.Evaluator[] polarityByValueEvaluators;
264
265 private final SetPrecisionRecall.Evaluator polarityEvaluator;
266
267 private final SetPrecisionRecall.Evaluator expressionEvaluator;
268
269 private final SetPrecisionRecall.Evaluator holderEvaluator;
270
271 private final SetPrecisionRecall.Evaluator targetEvaluator;
272
273 @Nullable
274 private OpinionPrecisionRecall score;
275
276 private Evaluator() {
277 final int numPolarities = Polarity.values().length;
278 this.polarityCMEvaluator = ConfusionMatrix.evaluator(numPolarities);
279 this.polarityByValueEvaluators = new SetPrecisionRecall.Evaluator[numPolarities];
280 this.polarityEvaluator = SetPrecisionRecall.evaluator();
281 this.expressionEvaluator = SetPrecisionRecall.evaluator();
282 this.holderEvaluator = SetPrecisionRecall.evaluator();
283 this.targetEvaluator = SetPrecisionRecall.evaluator();
284 this.score = null;
285 for (final Polarity polarity : Polarity.values()) {
286 this.polarityByValueEvaluators[polarity.ordinal()] = SetPrecisionRecall
287 .evaluator();
288 }
289 }
290
291 @SuppressWarnings("unchecked")
292 public synchronized Evaluator add(final Iterable<Opinion> goldOpinions,
293 final Iterable<Opinion> testOpinions) {
294
295
296 final Map<Set<Object>, Polarity> goldMap = getPolarityMap(goldOpinions, null);
297 final Map<Set<Object>, Polarity> testMap = getPolarityMap(testOpinions, null);
298 final Multimap<Set<Object>, Opinion> goldMultimap = indexOpinionsByExpression(goldOpinions);
299 final Multimap<Set<Object>, Opinion> testMultimap = indexOpinionsByExpression(testOpinions);
300
301
302 this.expressionEvaluator.add(goldMap.keySet(), testMap.keySet());
303
304
305 this.polarityEvaluator.add(goldMap, testMap);
306 for (final Polarity polarity : Polarity.values()) {
307 this.polarityByValueEvaluators[polarity.ordinal()].add(
308 getPolarityMap(goldOpinions, polarity),
309 getPolarityMap(testOpinions, polarity));
310 }
311
312
313 for (final Set<Object>[] pair : Util.align(Set.class, goldMap.keySet(),
314 testMap.keySet(), false, true, true, SetPrecisionRecall.matcher())) {
315 final Set<Object> g = pair[0];
316 final Set<Object> t = pair[1];
317 if (t != null && g != null) {
318 final Polarity gp = goldMap.get(g);
319 final Polarity tp = testMap.get(t);
320 this.polarityCMEvaluator.add(gp.ordinal(), tp.ordinal(), 1);
321 }
322 }
323
324
325 for (final Set<Object>[] pair : Util.align(Set.class, goldMap.keySet(),
326 testMap.keySet(), false, true, true, SetPrecisionRecall.matcher())) {
327 if (pair[0] != null && pair[1] != null) {
328 final Set<Object> g = pair[0];
329 final Set<Object> t = pair[1];
330 final Set<Set<Object>> goldHolders = Sets.newHashSet();
331 final Set<Set<Object>> goldTargets = Sets.newHashSet();
332 final Set<Set<Object>> testHolders = Sets.newHashSet();
333 final Set<Set<Object>> testTargets = Sets.newHashSet();
334 for (final Opinion opinion : goldMultimap.get(g)) {
335 goldHolders.add(getHolder(opinion));
336 goldTargets.add(getTarget(opinion));
337 }
338 for (final Opinion opinion : testMultimap.get(t)) {
339 testHolders.add(getHolder(opinion));
340 testTargets.add(getTarget(opinion));
341 }
342 this.holderEvaluator.add(goldHolders, testHolders);
343 this.targetEvaluator.add(goldTargets, testTargets);
344 }
345 }
346
347 return this;
348 }
349
350 public Evaluator add(final KAFDocument document, final String goldLabel,
351 final String testLabel) {
352
353 final Multimap<Integer, Opinion> goldMap = indexOpinionsBySentence(document
354 .getOpinions(goldLabel));
355 final Multimap<Integer, Opinion> testMap = indexOpinionsBySentence(document
356 .getOpinions(testLabel));
357
358 for (int i = 0; i < document.getNumSentences(); ++i) {
359 final Collection<Opinion> goldOpinions = goldMap.get(i);
360 final Collection<Opinion> testOpinions = testMap.get(i);
361 if (!goldOpinions.isEmpty() || !testOpinions.isEmpty()) {
362 add(goldOpinions, testOpinions);
363 }
364 }
365
366 return this;
367 }
368
369 public Evaluator add(final Iterable<KAFDocument> documents, final String goldLabel,
370 final String testLabel) {
371 StreamSupport.stream(documents.spliterator(), true).forEach(document -> {
372 Preconditions.checkNotNull(document);
373 add(document, goldLabel, testLabel);
374 });
375 return this;
376 }
377
378 public synchronized Evaluator add(final OpinionPrecisionRecall opr) {
379 this.score = null;
380 this.polarityCMEvaluator.add(opr.getPolarityCM());
381 for (int i = 0; i < this.polarityByValueEvaluators.length; ++i) {
382 this.polarityByValueEvaluators[i].add(opr.polaritySPRsByValue[i]);
383 }
384 this.polarityEvaluator.add(opr.getPolaritySPR());
385 this.expressionEvaluator.add(opr.getExpressionSPR());
386 this.holderEvaluator.add(opr.getHolderSPR());
387 this.targetEvaluator.add(opr.getTargetSPR());
388 return this;
389 }
390
391 public synchronized Evaluator add(final Evaluator evaluator) {
392 synchronized (evaluator) {
393 this.score = null;
394 this.polarityCMEvaluator.add(evaluator.polarityCMEvaluator);
395 for (int i = 0; i < this.polarityByValueEvaluators.length; ++i) {
396 this.polarityByValueEvaluators[i].add(evaluator.polarityByValueEvaluators[i]);
397 }
398 this.polarityEvaluator.add(evaluator.polarityEvaluator);
399 this.expressionEvaluator.add(evaluator.expressionEvaluator);
400 this.holderEvaluator.add(evaluator.holderEvaluator);
401 this.targetEvaluator.add(evaluator.targetEvaluator);
402 }
403 return this;
404 }
405
406 public synchronized OpinionPrecisionRecall getResult() {
407 if (this.score == null) {
408 final int polarities = this.polarityByValueEvaluators.length;
409 final ConfusionMatrix polarityCM = this.polarityCMEvaluator.getResult();
410 final SetPrecisionRecall[] polaritySPRsByValue = new SetPrecisionRecall[polarities];
411 for (int i = 0; i < polarities; ++i) {
412 polaritySPRsByValue[i] = this.polarityByValueEvaluators[i].getResult();
413 }
414 final SetPrecisionRecall polaritySPR = this.polarityEvaluator.getResult();
415 final SetPrecisionRecall expressionSPR = this.expressionEvaluator.getResult();
416 final SetPrecisionRecall holderSPR = this.holderEvaluator.getResult();
417 final SetPrecisionRecall targetSPR = this.targetEvaluator.getResult();
418 this.score = new OpinionPrecisionRecall(polarityCM, polaritySPRsByValue,
419 polaritySPR, expressionSPR, holderSPR, targetSPR);
420 }
421 return this.score;
422 }
423
424 }
425
426 }