1 package eu.fbk.dkm.pikes.resources.trec;
2
3 import com.google.common.base.Charsets;
4 import com.google.common.io.Files;
5 import eu.fbk.utils.core.CommandLine;
6 import ixa.kaflib.KAFDocument;
7 import org.joox.JOOX;
8 import org.slf4j.Logger;
9 import org.slf4j.LoggerFactory;
10 import org.w3c.dom.Document;
11 import org.w3c.dom.Element;
12 import org.xml.sax.SAXException;
13
14 import javax.xml.parsers.DocumentBuilder;
15 import javax.xml.parsers.DocumentBuilderFactory;
16 import javax.xml.parsers.ParserConfigurationException;
17 import java.io.ByteArrayInputStream;
18 import java.io.File;
19 import java.io.IOException;
20 import java.io.InputStream;
21 import java.text.SimpleDateFormat;
22 import java.util.Calendar;
23
24
25
26
27
28
29
30
31
32 public class CR {
33
34 private static final Logger LOGGER = LoggerFactory.getLogger(CR.class);
35 private static String DEFAULT_URL = "http://document/%s";
36 private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
37
38 public static void main(String[] args) {
39
40 try {
41
42 final CommandLine cmd = CommandLine
43 .parser()
44 .withName("cr-extractor")
45 .withHeader("Extract CR documents from TREC dataset and save them in NAF format")
46 .withOption("i", "input", "Input folder", "FOLDER", CommandLine.Type.DIRECTORY_EXISTING, true,
47 false, true)
48 .withOption("o", "output", "Output folder", "FOLDER", CommandLine.Type.DIRECTORY, true, false, true)
49 .withOption("u", "url-template", "URL template (with %d for the ID)", "URL",
50 CommandLine.Type.STRING, true, false, false)
51 .withLogger(LoggerFactory.getLogger("eu.fbk"))
52 .parse(args);
53
54 File inputDir = cmd.getOptionValue("input", File.class);
55
56 String urlTemplate = DEFAULT_URL;
57 if (cmd.hasOption("url-template")) {
58 urlTemplate = cmd.getOptionValue("url-template", String.class);
59 }
60
61 File outputDir = cmd.getOptionValue("output", File.class);
62 if (!outputDir.exists()) {
63 outputDir.mkdirs();
64 }
65
66 for (final File file : Files.fileTreeTraverser().preOrderTraversal(inputDir)) {
67 if (!file.isFile()) {
68 continue;
69 }
70 if (file.getName().startsWith(".")) {
71 continue;
72 }
73
74 String outputTemplate = outputDir.getAbsolutePath() + File.separator + file.getName();
75 File newFolder = new File(outputTemplate);
76 newFolder.mkdirs();
77
78 outputTemplate += File.separator + "NAF";
79 saveFile(file, outputTemplate, urlTemplate);
80 }
81 } catch (Exception e) {
82 CommandLine.fail(e);
83 }
84 }
85
86 private static void saveFile(File inputFile, String outputFilePattern, String urlTemplate)
87 throws IOException, SAXException, ParserConfigurationException {
88
89 LOGGER.info("Input file: {}", inputFile);
90
91 StringBuffer stringBuffer = new StringBuffer();
92 stringBuffer.append("<ROOT>\n");
93 stringBuffer.append(Files.toString(inputFile, Charsets.UTF_8));
94 stringBuffer.append("\n</ROOT>\n");
95
96 InputStream is = new ByteArrayInputStream(stringBuffer.toString().getBytes());
97 DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
98 DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
99 Document doc = dBuilder.parse(is);
100
101 doc.getDocumentElement().normalize();
102
103 int i = 0;
104 for (Element element : JOOX.$(doc).find("DOC")) {
105 Element docnoElement = JOOX.$(element).find("DOCNO").get(0);
106 Element dateElement = JOOX.$(element).find("DATE").get(0);
107 Element headlineElement = JOOX.$(element).find("TTL").get(0);
108
109
110 i++;
111 File outputFile = new File(outputFilePattern + "-" + i + ".naf");
112
113 String text = JOOX.$(element).find("TEXT").content();
114 if (text == null || text.length() == 0) {
115 LOGGER.error("TEXT is null");
116 continue;
117 }
118
119 String docno = "";
120 if (docnoElement != null) {
121 docno = docnoElement.getTextContent().trim();
122 }
123
124 String date = "";
125 if (dateElement != null) {
126 date = dateElement.getTextContent().trim();
127 }
128
129 String headline = "";
130 if (headlineElement != null) {
131 headline = headlineElement.getTextContent().trim();
132 }
133
134 if (docno.equals("")) {
135 LOGGER.error("DOCNO is empty");
136 }
137
138 String url = String.format(urlTemplate, docno);
139
140 headline = headline.replace('\n', ' ');
141 headline = headline.replaceAll("\\s+", " ");
142
143 Calendar.Builder builder = new Calendar.Builder();
144 try {
145 builder.setDate(1900 + Integer.parseInt(date.substring(0, 2)), Integer.parseInt(date.substring(2, 4)),
146 Integer.parseInt(date.substring(4)));
147 } catch (NumberFormatException e) {
148 LOGGER.error(e.getMessage());
149 }
150 Calendar calendar = builder.build();
151
152 text = headline + "\n\n" + text;
153
154 text = text.replaceAll("<TTL>.*</TTL>", "");
155 text = text.replaceAll("<FLD001>.*</FLD001>", "");
156 text = text.replaceAll("<FLD[0-9]{3}>.*?</FLD[0-9]{3}>", "");
157 text = text.replaceAll("</?[A-Za-z]+>", "");
158
159 KAFDocument document = new KAFDocument("en", "v3");
160 document.setRawText(text);
161
162 KAFDocument.FileDesc fileDesc = document.createFileDesc();
163 fileDesc.title = headline;
164 fileDesc.creationtime = sdf.format(calendar.getTime());
165 KAFDocument.Public aPublic = document.createPublic();
166 aPublic.uri = url;
167 aPublic.publicId = docno;
168
169 document.save(outputFile.getAbsolutePath());
170 }
171 }
172 }