1 package eu.fbk.dkm.pikes.rdf;
2
3 import java.io.File;
4 import java.io.IOException;
5 import java.io.InputStreamReader;
6 import java.io.StringReader;
7 import java.io.StringWriter;
8 import java.io.Writer;
9 import java.net.URL;
10 import java.util.Arrays;
11 import java.util.Collection;
12 import java.util.Comparator;
13 import java.util.List;
14 import java.util.Map;
15 import java.util.Set;
16 import java.util.concurrent.Callable;
17
18 import javax.annotation.Nullable;
19
20 import com.github.mustachejava.DefaultMustacheFactory;
21 import com.github.mustachejava.Mustache;
22 import com.github.mustachejava.util.LatchedWriter;
23 import com.google.common.base.Charsets;
24 import com.google.common.base.MoreObjects;
25 import com.google.common.base.Preconditions;
26 import com.google.common.base.Strings;
27 import com.google.common.collect.ImmutableList;
28 import com.google.common.collect.ImmutableMap;
29 import com.google.common.collect.ImmutableSet;
30 import com.google.common.collect.Iterables;
31 import com.google.common.collect.Lists;
32 import com.google.common.collect.Maps;
33 import com.google.common.collect.Ordering;
34 import com.google.common.collect.Sets;
35 import com.google.common.html.HtmlEscapers;
36 import com.google.common.io.Files;
37
38 import eu.fbk.utils.svm.Util;
39 import eu.fbk.dkm.pikes.rdf.vocab.*;
40 import org.eclipse.rdf4j.model.BNode;
41 import org.eclipse.rdf4j.model.Literal;
42 import org.eclipse.rdf4j.model.Model;
43 import org.eclipse.rdf4j.model.Resource;
44 import org.eclipse.rdf4j.model.Statement;
45 import org.eclipse.rdf4j.model.IRI;
46 import org.eclipse.rdf4j.model.Value;
47 import org.eclipse.rdf4j.model.impl.LinkedHashModel;
48 import org.eclipse.rdf4j.model.util.Models;
49 import org.eclipse.rdf4j.model.vocabulary.RDF;
50 import org.eclipse.rdf4j.model.vocabulary.XMLSchema;
51 import org.slf4j.Logger;
52 import org.slf4j.LoggerFactory;
53 import org.slf4j.MDC;
54
55 import ixa.kaflib.KAFDocument;
56 import ixa.kaflib.Term;
57
58 import eu.fbk.dkm.pikes.naflib.NafRenderUtils;
59 import eu.fbk.dkm.pikes.naflib.NafRenderUtils.Markable;
60 import eu.fbk.dkm.pikes.rdf.util.ModelUtil;
61 import eu.fbk.dkm.pikes.rdf.util.RDFGraphvizRenderer;
62 import eu.fbk.dkm.pikes.resources.NAFFilter;
63 import eu.fbk.dkm.pikes.resources.NAFUtils;
64 import eu.fbk.rdfpro.util.Hash;
65 import eu.fbk.rdfpro.util.IO;
66 import eu.fbk.rdfpro.util.Namespaces;
67 import eu.fbk.rdfpro.util.Options;
68 import eu.fbk.rdfpro.util.QuadModel;
69 import eu.fbk.rdfpro.util.Statements;
70 import eu.fbk.rdfpro.util.Tracker;
71
72 public class Renderer {
73
74 private static final Logger LOGGER = LoggerFactory.getLogger(Renderer.class);
75
76 public static final Set<IRI> DEFAULT_NODE_TYPES = ImmutableSet.of(KS_OLD.ENTITY, KS_OLD.ATTRIBUTE);
77
78 public static final Set<String> DEFAULT_NODE_NAMESPACES = ImmutableSet.of();
79
80 public static final Map<Object, String> DEFAULT_COLOR_MAP = ImmutableMap
81 .<Object, String>builder()
82 .put("node", "#F0F0F0")
83 .put(NWR.PERSON, "#FFC8C8")
84 .put(NWR.ORGANIZATION, "#FFFF84")
85 .put(NWR.LOCATION, "#A9C5EB")
86 .put(KS_OLD.ATTRIBUTE, "#EEBBEE")
87
88
89
90
91 .put(SUMO.PROCESS, "#CFE990")
92 .put(SUMO.RELATION, "#FFFFFF")
93 .put(OWLTIME.INTERVAL, "#B4D1B6")
94 .put(OWLTIME.DATE_TIME_INTERVAL, "#B4D1B6")
95 .put(OWLTIME.PROPER_INTERVAL, "#B4D1B6")
96 .put(NWR.MISC, "#D1BAA2")
97
98 .build();
99
100 public static final Map<Object, String> DEFAULT_STYLE_MAP = ImmutableMap.of();
101
102 public static final Mustache DEFAULT_TEMPLATE = loadTemplate("Renderer.html");
103
104 public static final List<String> DEFAULT_RANKED_NAMESPACES = ImmutableList.of(
105 "http://framebase.org/ns/", //
106 "http://www.newsreader-project.eu/ontologies/propbank/",
107 "http://www.newsreader-project.eu/ontologies/nombank/");
108
109 public static final Renderer DEFAULT = Renderer.builder().build();
110
111
112
113 private static final Mustache loadTemplate(final Object spec) {
114 Preconditions.checkNotNull(spec);
115 try {
116 if (spec instanceof Mustache) {
117 return (Mustache) spec;
118 }
119 final DefaultMustacheFactory factory = new DefaultMustacheFactory();
120
121 URL url = spec instanceof URL ? (URL) spec : null;
122 if (url == null) {
123 try {
124 url = Renderer.class.getResource(spec.toString());
125 } catch (final Throwable ex) {
126
127 }
128 }
129 if (url == null) {
130 final File file = spec instanceof File ? (File) spec : new File(spec.toString());
131 if (file.exists()) {
132 url = file.toURI().toURL();
133 }
134 }
135 if (url != null) {
136 return factory.compile(new InputStreamReader(url.openStream(), Charsets.UTF_8),
137 url.toString());
138 } else {
139 return factory.compile(new StringReader(spec.toString()),
140 Hash.murmur3(spec.toString()).toString());
141 }
142 } catch (final IOException ex) {
143 throw new IllegalArgumentException("Could not create Mustache template for " + spec);
144 }
145 }
146
147 private final Set<IRI> nodeTypes;
148
149 private final Set<String> nodeNamespaces;
150
151 private final Ordering<Value> valueComparator;
152
153 private final Ordering<Statement> statementComparator;
154
155 private final IRI denotedByProperty;
156
157 private final Map<Object, String> colorMap;
158
159 private final Map<Object, String> styleMap;
160
161 private Mustache template;
162
163 private Map<String, ?> templateParameters;
164
165 private Renderer(final Builder builder) {
166 this.nodeTypes = builder.nodeTypes == null ? DEFAULT_NODE_TYPES
167 : ImmutableSet.copyOf(builder.nodeTypes);
168 this.nodeNamespaces = builder.nodeNamespaces == null ? DEFAULT_NODE_NAMESPACES
169 : ImmutableSet.copyOf(builder.nodeNamespaces);
170 this.valueComparator = Ordering.from(Statements.valueComparator(Iterables.toArray(
171 builder.rankedNamespaces == null ? DEFAULT_RANKED_NAMESPACES
172 : builder.rankedNamespaces, String.class)));
173 this.statementComparator = new Ordering<Statement>() {
174
175 @Override
176 public int compare(final Statement first, final Statement second) {
177 final Comparator<Value> vc = Renderer.this.valueComparator;
178 int result = vc.compare(first.getSubject(), second.getSubject());
179 if (result == 0) {
180 result = vc.compare(first.getPredicate(), second.getPredicate());
181 if (result == 0) {
182 result = vc.compare(first.getObject(), second.getObject());
183 if (result == 0) {
184 result = vc.compare(first.getContext(), second.getContext());
185 }
186 }
187 }
188 return result;
189 }
190
191 };
192 this.denotedByProperty = MoreObjects.firstNonNull(builder.denotedByProperty,
193 GAF.DENOTED_BY);
194 this.colorMap = builder.colorMap == null ? DEFAULT_COLOR_MAP : ImmutableMap
195 .copyOf(builder.colorMap);
196 this.styleMap = builder.styleMap == null ? DEFAULT_STYLE_MAP : ImmutableMap
197 .copyOf(builder.styleMap);
198 this.template = MoreObjects.firstNonNull(builder.template, DEFAULT_TEMPLATE);
199 this.templateParameters = builder.templateParameters;
200 }
201
202 public void renderAll(final Appendable out, final KAFDocument document, final Model model,
203 @Nullable final Object template, @Nullable final Map<String, ?> templateParameters)
204 throws IOException {
205
206 final long ts = System.currentTimeMillis();
207 final KAFDocument doc = document;
208 final long[] times = new long[8];
209
210 final List<Map<String, Object>> sentencesModel = Lists.newArrayList();
211 for (int i = 1; i <= doc.getNumSentences(); ++i) {
212 final Map<String, Object> sm = Maps.newHashMap();
213 sm.put("id", i);
214 sm.put("markup", new Renderable(doc, model, i, times, Renderable.SENTENCE_TEXT));
215 sm.put("parsing", new Renderable(doc, model, i, times, Renderable.SENTENCE_PARSING));
216 sm.put("graph", new Renderable(doc, model, i, times, Renderable.SENTENCE_GRAPH));
217 sentencesModel.add(sm);
218 }
219
220 final Map<String, Object> documentModel = Maps.newHashMap();
221 documentModel.put("title", doc.getPublic().uri);
222 documentModel.put("sentences", sentencesModel);
223 documentModel.put("metadata", new Renderable(doc, model, -1, times, Renderable.METADATA));
224 documentModel.put("mentions", new Renderable(doc, model, -1, times, Renderable.MENTIONS));
225 documentModel.put("triples", new Renderable(doc, model, -1, times, Renderable.TRIPLES));
226 documentModel.put("graph", new Renderable(doc, model, -1, times, Renderable.GRAPH));
227 documentModel.put("naf", new Renderable(doc, model, -1, times, Renderable.NAF));
228
229 documentModel.putAll(this.templateParameters);
230 if (templateParameters != null) {
231 documentModel.putAll(templateParameters);
232 }
233
234 final Mustache actualTemplate = template != null ? loadTemplate(template) : this.template;
235 if (out instanceof Writer) {
236 Writer outWriter;
237 outWriter = actualTemplate.execute((Writer) out, documentModel);
238 if (outWriter instanceof LatchedWriter) {
239 ((LatchedWriter) outWriter).await();
240 outWriter.flush();
241 }
242 } else {
243 final StringWriter writer = new StringWriter();
244 actualTemplate.execute(writer, documentModel).close();
245 out.append(writer.toString());
246 }
247
248 if (LOGGER.isDebugEnabled()) {
249 LOGGER.debug("Done in {} ms ({} text, {} parsing, {} graphs, {} metadata, "
250 + "{} mentions, {} triples, {} naf)", System.currentTimeMillis() - ts,
251 times[Renderable.SENTENCE_TEXT], times[Renderable.SENTENCE_PARSING],
252 times[Renderable.SENTENCE_GRAPH] + times[Renderable.GRAPH],
253 times[Renderable.METADATA], times[Renderable.MENTIONS],
254 times[Renderable.TRIPLES], times[Renderable.NAF]);
255 }
256 }
257
258 public void renderGraph(final Appendable out, final QuadModel model, final Algorithm algorithm)
259 throws IOException {
260 RDFGraphvizRenderer.builder().withNodeNamespaces(this.nodeNamespaces)
261 .withNodeTypes(this.nodeTypes).withValueComparator(this.valueComparator)
262 .withCollapsedProperties(ImmutableSet.of(this.denotedByProperty))
263 .withColorMap(this.colorMap).withStyleMap(this.styleMap)
264 .withGraphvizCommand(algorithm.name().toLowerCase()).build().emitSVG(out, model);
265 }
266
267 public void renderText(final Appendable out, final KAFDocument document,
268 final Iterable<Term> terms, final Model model) throws IOException {
269 final List<Term> termList = Ordering.from(Term.OFFSET_COMPARATOR).sortedCopy(terms);
270 NafRenderUtils.renderText(out, document, terms,
271 extractMarkables(termList, model, this.colorMap));
272 }
273
274 public void renderParsing(final Appendable out, final KAFDocument document,
275 @Nullable final Model model, final int sentence) throws IOException {
276 NafRenderUtils.renderParsing(out, document, sentence, true, true,
277 extractMarkables(document.getTermsBySent(sentence), model, this.colorMap));
278 }
279
280 public void renderProperties(final Appendable out, final Model model, final Resource node,
281 final boolean emitID, final IRI... excludedProperties) throws IOException {
282
283 final Set<Resource> seen = Sets.newHashSet(node);
284 renderPropertiesHelper(out, model, node, emitID, seen,
285 ImmutableSet.copyOf(excludedProperties));
286 }
287
288 private void renderPropertiesHelper(final Appendable out, final Model model,
289 final Resource node, final boolean emitID, final Set<Resource> seen,
290 final Set<IRI> excludedProperties) throws IOException {
291
292
293 out.append("<table class=\"properties table table-condensed\">\n<tbody>\n");
294
295
296 if (emitID) {
297 out.append("<tr><td><a>ID</a>:</td><td>");
298 renderObject(out, node, model);
299 out.append("</td></tr>\n");
300 }
301
302
303 for (final IRI pred : this.valueComparator.sortedCopy(model.filter(node, null, null)
304 .predicates())) {
305 if (excludedProperties.contains(pred)) {
306 continue;
307 }
308 out.append("<tr><td>");
309 renderObject(out, pred, model);
310 out.append(":</td><td>");
311 final List<Resource> nested = Lists.newArrayList();
312 String separator = "";
313 for (final Value obj : this.valueComparator.sortedCopy(model.filter(node, pred, null)
314 .objects())) {
315 if (obj instanceof Literal || model.filter((Resource) obj, null, null).isEmpty()) {
316 out.append(separator);
317 renderObject(out, obj, model);
318 separator = ", ";
319 } else {
320 nested.add((Resource) obj);
321 }
322 }
323 out.append("".equals(separator) ? "" : "<br/>");
324 for (final Resource obj : nested) {
325 out.append(separator);
326 if (seen.add(obj)) {
327 renderPropertiesHelper(out, model, obj, true, seen, excludedProperties);
328 } else {
329 renderObject(out, obj, model);
330 }
331 }
332 out.append("</td></tr>\n");
333 }
334
335
336 out.append("</tbody>\n</table>\n");
337 }
338
339 public void renderTriplesTable(final Appendable out, final Model model) throws IOException {
340
341 out.append("<table class=\"table table-condensed datatable\">\n<thead>\n");
342 out.append("<tr><th width='25%' class='col-ts'>");
343 out.append(shorten(RDF.SUBJECT));
344 out.append("</th><th width='25%' class='col-tp'>");
345 out.append(shorten(RDF.PREDICATE));
346 out.append("</th><th width='25%' class='col-to'>");
347 out.append(shorten(RDF.OBJECT));
348 out.append("</th><th width='25%' class='col-te'>");
349 out.append(shorten(KS_OLD.EXPRESSED_BY));
350 out.append("</th></tr>\n");
351 out.append("</thead>\n<tbody>\n");
352
353 for (final Statement statement : this.statementComparator.sortedCopy(model)) {
354 if (statement.getContext() != null) {
355 out.append("<tr><td>");
356 renderObject(out, statement.getSubject(), model);
357 out.append("</td><td>");
358 renderObject(out, statement.getPredicate(), model);
359 out.append("</td><td>");
360 renderObject(out, statement.getObject(), model);
361 out.append("</td><td>");
362 String separator = "";
363 for (final Value mentionID : model.filter(statement.getContext(), KS_OLD.EXPRESSED_BY,
364 null).objects()) {
365 final String extent = Models.objectString(model.filter((Resource) mentionID, NIF.ANCHOR_OF, null)).get();
366 out.append(separator);
367 renderObject(out, mentionID, model);
368 out.append(" '").append(escape(extent)).append("'");
369 separator = "<br/>";
370 }
371 out.append("</ol></td></tr>\n");
372 }
373 }
374
375 out.append("</tbody>\n</table>\n");
376 }
377
378 public void renderMentionsTable(final Appendable out, final Model model) throws IOException {
379
380 out.append("<table class=\"table table-condensed datatable\">\n<thead>\n");
381 out.append("<tr><th width='12%' class='col-mi'>id</th><th width='18%' class='col-ma'>");
382 out.append(shorten(NIF.ANCHOR_OF));
383 out.append("</th><th width='11%' class='col-mt'>");
384 out.append(shorten(RDF.TYPE));
385 out.append("</th><th width='18%' class='col-mo'>mention attributes</th><th width='11%' class='col-md'>");
386 out.append(shorten(GAF.DENOTED_BY)).append("<sup>-1</sup>");
387 out.append("</th><th width='30%' class='col-me'>");
388 out.append(shorten(KS_OLD.EXPRESSED_BY)).append("<sup>-1</sup>");
389 out.append("</th></tr>\n</thead>\n<tbody>\n");
390
391 for (final Resource mentionID : this.valueComparator.sortedCopy(ModelUtil
392 .getMentions(QuadModel.wrap(model)))) {
393 out.append("<tr><td>");
394 renderObject(out, mentionID, model);
395 out.append("</td><td>");
396 out.append(Models.objectString(model.filter(mentionID, NIF.ANCHOR_OF, null)).get());
397 out.append("</td><td>");
398 renderObject(out, model.filter(mentionID, RDF.TYPE, null).objects(), model);
399 out.append("</td><td>");
400 final Model mentionModel = new LinkedHashModel();
401 for (final Statement statement : model.filter(mentionID, null, null)) {
402 final IRI pred = statement.getPredicate();
403 if (!NIF.BEGIN_INDEX.equals(pred) && !NIF.END_INDEX.equals(pred)
404 && !NIF.ANCHOR_OF.equals(pred) && !RDF.TYPE.equals(pred)
405 && !KS_OLD.MENTION_OF.equals(pred)) {
406 mentionModel.add(statement);
407 }
408 }
409 if (!mentionModel.isEmpty()) {
410 renderProperties(out, mentionModel, mentionID, false);
411 }
412 out.append("</td><td>");
413 renderObject(out, model.filter(null, GAF.DENOTED_BY, mentionID).subjects(), model);
414 out.append("</td><td><ol>");
415 for (final Resource factID : model.filter(null, KS_OLD.EXPRESSED_BY, mentionID).subjects()) {
416 for (final Statement statement : model.filter(null, null, null, factID)) {
417 out.append("<li>");
418 renderObject(out, statement.getSubject(), model);
419 out.append(", ");
420 renderObject(out, statement.getPredicate(), model);
421 out.append(", ");
422 renderObject(out, statement.getObject(), model);
423 out.append("</li>");
424 }
425 }
426 out.append("</ol></td></tr>\n");
427 }
428
429 out.append("</tbody>\n</table>\n");
430 }
431
432 public void renderObject(final Appendable out, final Object object, @Nullable final Model model)
433 throws IOException {
434
435 if (object instanceof IRI) {
436 final IRI uri = (IRI) object;
437 out.append("<a>").append(shorten(uri)).append("</a>");
438
439 } else if (object instanceof Literal) {
440 final Literal literal = (Literal) object;
441 out.append("<span");
442 if (literal.getLanguage().isPresent()) {
443 out.append(" title=\"@").append(literal.getLanguage().get()).append("\"");
444 } else if (!literal.getDatatype().equals(XMLSchema.STRING)) {
445 out.append(" title=\"").append(shorten(literal.getDatatype())).append("\"");
446 }
447 out.append(">").append(literal.stringValue()).append("</span>");
448
449 } else if (object instanceof BNode) {
450 final BNode bnode = (BNode) object;
451 out.append("_:").append(bnode.getID());
452
453 } else if (object instanceof Iterable<?>) {
454 String separator = "";
455 for (final Object element : (Iterable<?>) object) {
456 out.append(separator);
457 renderObject(out, element, model);
458 separator = "<br/>";
459 }
460
461 } else if (object != null) {
462 out.append(object.toString());
463 }
464 }
465
466 private static List<Markable> extractMarkables(final List<Term> terms, final Model model,
467 final Map<Object, String> colorMap) {
468
469 final int[] offsets = new int[terms.size()];
470 for (int i = 0; i < terms.size(); ++i) {
471 offsets[i] = terms.get(i).getOffset();
472 }
473
474 final List<Markable> markables = Lists.newArrayList();
475 for (final Statement stmt : model.filter(null, GAF.DENOTED_BY, null)) {
476 final Resource instance = stmt.getSubject();
477 final String color = select(colorMap,
478 model.filter(instance, RDF.TYPE, null).objects(), null);
479 if (stmt.getObject() instanceof IRI && color != null) {
480 final IRI mentionIRI = (IRI) stmt.getObject();
481 final String name = mentionIRI.getLocalName();
482 if (name.indexOf(';') < 0) {
483 final int index = name.indexOf(',');
484 final int start = Integer.parseInt(name.substring(5, index));
485 final int end = Integer.parseInt(name.substring(index + 1));
486 final int s = Arrays.binarySearch(offsets, start);
487 if (s >= 0) {
488 int e = s;
489 while (e < offsets.length && offsets[e] < end) {
490 ++e;
491 }
492 markables.add(new Markable(ImmutableList.copyOf(terms.subList(s, e)),
493 color));
494 }
495 }
496 }
497 }
498
499 return markables;
500 }
501
502 private static String select(final Map<Object, String> map,
503 final Iterable<? extends Value> keys, final String defaultColor) {
504 String color = null;
505 for (final Value key : keys) {
506 if (key instanceof IRI) {
507 final String mappedColor = map.get(key);
508 if (mappedColor != null) {
509 if (color == null) {
510 color = mappedColor;
511 } else {
512 break;
513 }
514 }
515 }
516 }
517 return color != null ? color : defaultColor;
518 }
519
520 private static String escape(final String string) {
521 return HtmlEscapers.htmlEscaper().escape(string);
522 }
523
524 @Nullable
525 private static String shorten(@Nullable final IRI uri) {
526 if (uri == null) {
527 return null;
528 }
529 final String prefix = Namespaces.DEFAULT.prefixFor(uri.getNamespace());
530 if (prefix != null) {
531 return prefix + ':' + uri.getLocalName();
532 }
533 return "<../" + uri.getLocalName() + ">";
534 }
535
536 public static Builder builder() {
537 return new Builder();
538 }
539
540 public static final class Builder {
541
542 @Nullable
543 private Iterable<? extends IRI> nodeTypes;
544
545 @Nullable
546 private Iterable<? extends String> nodeNamespaces;
547
548 @Nullable
549 private Iterable<? extends String> rankedNamespaces;
550
551 @Nullable
552 private IRI denotedByProperty;
553
554 @Nullable
555 private Map<Object, String> colorMap;
556
557 @Nullable
558 private Map<Object, String> styleMap;
559
560 @Nullable
561 private Mustache template;
562
563 private final Map<String, Object> templateParameters;
564
565 Builder() {
566 this.templateParameters = Maps.newHashMap();
567 }
568
569 public Builder withProperties(final Map<?, ?> properties, @Nullable final String prefix) {
570 final String p = prefix == null ? "" : prefix.endsWith(".") ? prefix : prefix + ".";
571 for (final Map.Entry<?, ?> entry : properties.entrySet()) {
572 if (entry.getKey() != null && entry.getValue() != null
573 && entry.getKey().toString().startsWith(p)) {
574 final String name = entry.getKey().toString().substring(p.length());
575 final String value = Strings.emptyToNull(entry.getValue().toString());
576 if ("template".equals(name)) {
577 withTemplate(value);
578 } else if (name.startsWith("template.")) {
579 withTemplateParameter(name.substring("template.".length()), value);
580 }
581 }
582 }
583 return this;
584 }
585
586 public Builder withNodeTypes(@Nullable final Iterable<? extends IRI> nodeTypes) {
587 this.nodeTypes = nodeTypes;
588 return this;
589 }
590
591 public Builder withNodeNamespaces(@Nullable final Iterable<? extends String> nodeNamespaces) {
592 this.nodeNamespaces = nodeNamespaces;
593 return this;
594 }
595
596 public Builder withRankedNamespaces(
597 @Nullable final Iterable<? extends String> rankedNamespaces) {
598 this.rankedNamespaces = rankedNamespaces;
599 return this;
600 }
601
602 public Builder withDenotedByProperty(@Nullable final IRI denotedByProperty) {
603 this.denotedByProperty = denotedByProperty;
604 return this;
605 }
606
607 public Builder withColorMap(@Nullable final Map<Object, String> colorMap) {
608 this.colorMap = colorMap;
609 return this;
610 }
611
612 public Builder withStyleMap(@Nullable final Map<Object, String> styleMap) {
613 this.styleMap = styleMap;
614 return this;
615 }
616
617 public Builder withTemplate(@Nullable final Object template) {
618 this.template = template == null ? null : loadTemplate(template);
619 return this;
620 }
621
622 public Builder withTemplateParameter(final String name, @Nullable final Object value) {
623 this.templateParameters.put(name, value);
624 return this;
625 }
626
627 public Renderer build() {
628 return new Renderer(this);
629 }
630
631 }
632
633 public static enum Algorithm {
634
635 DOT,
636
637 NEATO,
638
639 FDP,
640
641 SFDP,
642
643 TWOPI,
644
645 CIRCO
646
647 }
648
649 private final class Renderable implements Callable<String> {
650
651 static final int SENTENCE_TEXT = 0;
652
653 static final int SENTENCE_PARSING = 1;
654
655 static final int SENTENCE_GRAPH = 2;
656
657 static final int METADATA = 3;
658
659 static final int MENTIONS = 4;
660
661 static final int TRIPLES = 5;
662
663 static final int GRAPH = 6;
664
665 static final int NAF = 7;
666
667 private final KAFDocument document;
668
669 private final Model model;
670
671 private final int sentenceID;
672
673 @Nullable
674 private final long[] times;
675
676 private final int type;
677
678 private Renderable(final KAFDocument document, final Model model, final int sentenceID,
679 @Nullable final long[] times, final int type) {
680 this.document = document;
681 this.model = model;
682 this.sentenceID = sentenceID;
683 this.times = times;
684 this.type = type;
685 }
686
687 @Override
688 public String call() throws Exception {
689 final long ts = System.currentTimeMillis();
690 try {
691 if (this.type == NAF) {
692 return this.document.toString();
693 } else {
694 final StringBuilder builder = new StringBuilder(128 * 1024);
695 if (this.type == SENTENCE_TEXT) {
696 renderText(builder, this.document,
697 this.document.getTermsBySent(this.sentenceID), this.model);
698 } else if (this.type == SENTENCE_PARSING) {
699 renderParsing(builder, this.document, this.model, this.sentenceID);
700 } else if (this.type == SENTENCE_GRAPH) {
701 int begin = Integer.MAX_VALUE;
702 int end = Integer.MIN_VALUE;
703 for (final Term term : this.document.getSentenceTerms(this.sentenceID)) {
704 begin = Math.min(begin, NAFUtils.getBegin(term));
705 end = Math.max(end, NAFUtils.getEnd(term));
706 }
707 final QuadModel sentenceModel = ModelUtil.getSubModel(
708 QuadModel.wrap(this.model),
709 ModelUtil.getMentions(QuadModel.wrap(this.model), begin, end));
710 renderGraph(builder, sentenceModel, Algorithm.NEATO);
711 } else if (this.type == METADATA) {
712 renderProperties(builder, this.model,
713 Statements.VALUE_FACTORY.createIRI(this.document.getPublic().uri), true, KS_OLD.HAS_MENTION);
714 } else if (this.type == MENTIONS) {
715 renderMentionsTable(builder, this.model);
716 } else if (this.type == TRIPLES) {
717 renderTriplesTable(builder, this.model);
718 } else if (this.type == GRAPH) {
719 renderGraph(builder, QuadModel.wrap(this.model), Algorithm.NEATO);
720 } else {
721 throw new Error("Unexpected rendering type " + this.type);
722 }
723 return builder.toString();
724 }
725 } catch (final Throwable ex) {
726 LOGGER.error("Renderable task failed", ex);
727 throw ex;
728 } finally {
729 if (this.times != null) {
730 synchronized (this.times) {
731 this.times[this.type] += System.currentTimeMillis() - ts;
732 }
733 }
734 }
735 }
736
737 }
738
739 static final class Runner implements Runnable {
740
741 private final List<File> inputFiles;
742
743 private final List<File> outputFiles;
744
745 private final RDFGenerator generator;
746
747 @Nullable
748 private final Mustache template;
749
750 private Runner(final List<File> inputFiles, final List<File> outputFiles,
751 final RDFGenerator generator, @Nullable final String template) {
752 this.inputFiles = inputFiles;
753 this.outputFiles = outputFiles;
754 this.generator = generator;
755 this.template = template == null ? null : loadTemplate(template);
756 }
757
758 private static void addFiles(final Collection<File> inputFiles,
759 final Collection<File> outputFiles, final File input, final File output,
760 final String format, final boolean recursive) {
761 if (input.isFile()) {
762 inputFiles.add(input);
763 outputFiles
764 .add(new File(output.getAbsolutePath() + "/" + input.getName() + format));
765 } else {
766 for (final File entry : input.listFiles()) {
767 final String name = entry.getName();
768 if (entry.isDirectory() && recursive || entry.isFile()
769 && (name.endsWith(".naf") || name.endsWith(".naf.xml"))) {
770 addFiles(inputFiles, outputFiles, entry, new File(output.getAbsolutePath()
771 + "/" + input.getName()), format, recursive);
772 }
773 }
774 }
775 }
776
777 static Runner create(final String name, final String... args) {
778
779 final Options options = Options.parse(
780 "r,recursive|f,format!|t,template!|d,directory!|m,merge|n,normalize|+", args);
781
782 final String template = options.getOptionArg("t", String.class);
783 String format = options.getOptionArg("f", String.class, ".html.gz");
784 format = format.startsWith(".") ? format : "." + format;
785 final boolean merge = options.hasOption("m");
786 final boolean normalize = options.hasOption("n");
787
788 File outputDir = options.getOptionArg("d", File.class);
789 if (outputDir == null) {
790 outputDir = new File(System.getProperty("user.dir"));
791 } else if (!outputDir.exists()) {
792 throw new IllegalArgumentException("Directory '" + outputDir + "' does not exist");
793 }
794
795 final boolean recursive = options.hasOption("r");
796 final List<File> inputFiles = Lists.newArrayList();
797 final List<File> outputFiles = Lists.newArrayList();
798 for (final File file : options.getPositionalArgs(File.class)) {
799 if (!file.exists()) {
800 throw new IllegalArgumentException("File/directory '" + file
801 + "' does not exist");
802 }
803 addFiles(inputFiles, outputFiles, file, outputDir, format, recursive);
804 }
805
806 final RDFGenerator generator = RDFGenerator.builder()
807 .withProperties(Util.PROPERTIES, "eu.fbk.dkm.pikes.cmd.RDFGenerator")
808 .withMerging(merge).withNormalization(normalize).build();
809
810 return new Runner(inputFiles, outputFiles, generator, template);
811 }
812
813 @Override
814 public void run() {
815
816 LOGGER.info("Rendering {} NAF files to HTML", this.inputFiles.size());
817
818 final NAFFilter filter = NAFFilter.builder()
819 .withProperties(Util.PROPERTIES, "eu.fbk.dkm.pikes.cmd.NAFFilter").build();
820
821 final Renderer renderer = Renderer.DEFAULT;
822
823 final Tracker tracker = new Tracker(LOGGER, null,
824 "Processed %d NAF files (%d NAF/s avg)",
825 "Processed %d NAF files (%d NAF/s, %d NAF/s avg)");
826
827 int succeeded = 0;
828 tracker.start();
829 for (int i = 0; i < this.inputFiles.size(); ++i) {
830 final File inputFile = this.inputFiles.get(i);
831 final File outputFile = this.outputFiles.get(i);
832 LOGGER.debug("Processing {} ...", inputFile);
833 MDC.put("context", inputFile.getName());
834 try {
835 final KAFDocument document = KAFDocument.createFromFile(inputFile);
836 filter.filter(document);
837 final Model model = this.generator.generate(document, null);
838 Files.createParentDirs(outputFile);
839 try (Writer writer = IO.utf8Writer(IO.write(outputFile.getAbsolutePath()))) {
840 renderer.renderAll(writer, document, model, this.template, null);
841 }
842 ++succeeded;
843 } catch (final Throwable ex) {
844 LOGGER.error("Processing failed for " + inputFile, ex);
845 } finally {
846 MDC.remove("context");
847 }
848 tracker.increment();
849 }
850 tracker.end();
851
852 LOGGER.info("Successfully rendered {}/{} files", succeeded, this.inputFiles.size());
853 }
854
855 }
856
857 }