* irrelevant element returned by evaluated system, not considered as false positive
| Sentence | Gold | FRED | PIKES |
|---|---|---|---|
| S1 | :approval | :approval_1 | :approval |
| :become | :become_1 | :become | |
| :challenge | :challenge_1 | :challenge | |
| :countries | :united_states_1_countries | :country | |
| :explicit | :Explicit | :explicit | |
| :extremists | :islamic_extremists_1 | :extremist | |
| :fighting | :fighting_1 | :fighting | |
| :forces | :force_1 | :force | |
| :group | :group_1 | :group | |
| :lone | :lone | ||
| :not | <attr:not-1r> | ||
| :other | :other | ||
| :posing | :pose_1 | :pose | |
| :qaeda | :Al_qaeda | :Al_Qaeda | |
| :rebel | :rebel_1 | :rebel | |
| :rebels | :rebel_2 | ||
| :stamp | :stamp_1 | :stamp | |
| :stark | :Stark | <attr:blunt-4a> | |
| :states | :united_states_1_States | :United_States | |
| :support | :support | ||
| :support_1 | :support_1 | ||
| :syrian | :Syrian | ||
| :uprising | :uprising_1 | :uprising | |
| :want | :want_1 | :want | |
| :Effective | |||
| :Most | |||
| :but_not_1 | |||
| <attr:about-7r> | |||
| <entity:islamic> | |||
| :effective | |||
| :one | |||
| S2 | :abroad | :Abroad | <attr:abroad-1r> |
| :donors | :donor_1 | :donor | |
| :flows | :flow_1 | :flow | |
| :front | :Nusra_front | :Nusra_Front | |
| :group | :group_1 | :group | |
| :like-minded | :Like-minded | :like-mind | |
| :money | :money_1 | <entity:money> | |
| S3 | :bases | :military_bases_1 | :basis |
| :battalions | :battalion_1 | :battalion | |
| :boldness | :boldness_1_boldness | :boldness | |
| :capture | :capture_1 | :capture | |
| :fields | :oil_fields_1 | :field | |
| :fighters | :fighter_1 | :fighter | |
| :fortified | :fortify_1 | :fortify | |
| :have | :have_1 | :have | |
| :its | :neuter_1 | :its | |
| :lead | :Lead | :lead | |
| :military | :military | ||
| :minority | :minority_1 | :minority | |
| :oil | :oil | ||
| :other | :other | ||
| :positions | :storm_1_positions | :position | |
| :rebels | :rebel_1 | :rebel | |
| :skill | :boldness_1_skill | :skill | |
| :small | :Small | :small | |
| :storm | :storm_1_storm | :storm | |
| S4 | :attract | :attract_1 | :attract |
| :fighters | :fighter_1 | :fighter | |
| :gather | :gather_1 | :gather | |
| :mount | :mount_1 | :mount | |
| :successes | :success_1 | :success | |
| :their | :thing_1 | :they | |
| :they | :thing_2 | :they_2 | |
| :weapons | :weapon_1 | :weapon | |
| :More_more_1 | |||
| :More_more_2 | |||
| :more | |||
| :more_2 | |||
| S5 | :contributed | :contribute_1 | :contribute |
| :direct | :Direct | :direct | |
| :fighters | :fighter_1_fighters | :fighter | |
| :former | :Former | :former | |
| :group | :group_1 | :group | |
| :insurgents | :iraqi_insurgents_1 | :insurgent | |
| :iraqi_1 | :Iraqi_Iraqi_1 | :Iraqi | |
| :iraqi_2 | :Iraqi_Iraqi_2 | <entity:iraqi> | |
| :is | :be | ||
| :officials | :official_1 | :official | |
| :offshoot | :offshoot | ||
| :qaeda | :Al_qaeda_in_iraq | :Al_Qaeda | |
| :say | :say_1 | :say | |
| :veteran | :Veteran | :veteran | |
| :weapons | :fighter_1_weapons | :weapon | |
| :which | |||
| :Iraq | |||
| :have | |||
| S6 | :brothers | :brother_1 | :brother |
| :efforts | :effort_1 | :effort | |
| :favor | :favor_1 | :favor | |
| :fought | :fight_1 | :fight | |
| :front | :Nusra_front | :Nusra_Front | |
| :he | :Al_qaeda_in_iraq_he | :he | |
| :helped | :help | ||
| :iraq_1 | :Iraq_Iraq_1 | :Iraq | |
| :iraq_2 | :Iraq_Iraq_2 | :Iraq_2 | |
| :is | |||
| :lands | :land_1 | :land | |
| :lead | :lead_1 | :lead | |
| :our | :person_1 | :we | |
| :qaeda | :Al_qaeda_in_iraq_Qaeda | :Al_Qaeda | |
| :returning | :return_1 | :return | |
| :said_1 | :say_1 | :say_2 | |
| :said_2 | :say_2 | :say | |
| :simple | :Simple | <attr:simple-1a> | |
| :syria | :Syria_Syria | :Syria | |
| :syrian | :Syrian, :Syria_Syrian | :Syrian | |
| :this | :thing_1 | ||
| :us | :person_2 | :we_2 | |
| :veteran | :veteran_1 | :veteran | |
| :way | :way_1 | :way | |
| :who | :who | ||
| :Just | |||
| <attr:merely-1r> | |||
| S7 | :al-assad | :Bashar_al-assad_al-Assad | :Bashar_al-Assad |
| :group | :group_1 | :group | |
| :hopes | :hope_1 | :hope | |
| :inheriting | :inherit_1 | :inherit | |
| :isolate | :isolate_1 | :isolate | |
| :it | :time_1_it | :it | |
| :president | :Syria_president, :Bashar_al-assad_president | :president | |
| :prevent | :prevent_1 | :prevent | |
| :running | :run_1 | :run | |
| :sensing | :sense_1 | :sense | |
| :states | :united_states_1 | :United_States | |
| :syria_1 | :Syria_Syria_1 | :Syria | |
| :syria_2 | :Syria_Syria_2 | :Syria_2 | |
| :time | :time_1_time | :time | |
| :Out | |||
| :be_1 | |||
| :may | |||
| :out | |||
| S8 | :alternative | :alternative | |
| :americans | :Americans | :Americans | |
| :blacklist | :blacklist_1 | :blacklist | |
| :dealings | :dealings_1 | :dealings | |
| :europe | :Europe | :Europe | |
| :financial | :Financial | <attr:fiscal-1a> | |
| :front | :Nusra_front | :Nusra_Front | |
| :government | :government_1 | :government | |
| :group | :group_1 | :group | |
| :have | :have_1 | :have | |
| :illegal | :Illegal | :illegal | |
| :it_1 | :it | ||
| :making | :make_1 | :make | |
| :opposition | :syrian_opposition_1 | :opposition | |
| :organization | :organization_1 | :organization | |
| :organize | :organize_1 | :organize | |
| :plans | :plan_1 | :plan | |
| :prompting | :prompt_1 | :prompt | |
| :pushes | :push_1 | :push | |
| :sanctions | :sanction_1 | :sanction | |
| :states | :United_states | :United_States | |
| :syrian | <entity:syrian> | ||
| :terrorist | :Terrorist | :terrorist | |
| :viable | :Viable | <attr:feasible-1a> | |
| :Similar | |||
| :it_2 | |||
| :similar | |||
| Results w.r.t. gold standard | f1=0.898 p=0.930 r=0.869 tp=119 fp=9 fn=18 | f1=0.940 p=0.911 r=0.971 tp=133 fp=13 fn=4 | |
| Results w.r.t. union of correct answers | f1=0.905 p=0.930 r=0.881 tp=119 fp=9 fn=16 | f1=0.947 p=0.911 r=0.985 tp=133 fp=13 fn=2 | |