* irrelevant element returned by evaluated system, not considered as false positive
| Sentence | Gold | FRED | PIKES |
|---|---|---|---|
| S1 | (:approval, :explicit) | ||
| (:approval, :group) | (:approval, :group) | ||
| (:approval, :qaeda) | (:Al_qaeda, :approval_1) | (:Al_Qaeda, :approval) | |
| (:become, :forces) | |||
| (:become, :group) | (:become_1, :group_1) | (:become, :group) | |
| (:challenge, :countries) | (:challenge, :country) | ||
| (:challenge, :group) | (:challenge, :group) | ||
| (:challenge, :posing) | (:challenge_1, :pose_1) | (:challenge, :pose) | |
| (:challenge, :stark) | (:Stark, :challenge_1) | (<attr:blunt-4a>, :challenge) | |
| (:challenge, :states) | (:United_States, :challenge) | ||
| (:countries, :other) | (:country, :other) | ||
| (:countries, :posing) | (:pose_1, :united_states_1_countries) | ||
| (:countries, :support) | (:country, :support) | ||
| (:countries, :support_1) | (:support_1, :united_states_1_countries) | ||
| (:countries, :want) | (:united_states_1_countries, :want_1) | (:country, :want) | |
| (:extremists, :support_1) | (:islamic_extremists_1, :support_1) | ||
| (:fighting, :forces) | (:fighting_1, :force_1) | (:fighting, :force) | |
| (:forces, :uprising) | (:force_1, :uprising_1) | (:force, :uprising) | |
| (:group, :lone) | (:group, :lone) | ||
| (:group, :posing) | (:group_1, :pose_1) | (:group, :pose) | |
| (:group, :rebel) | (:group_1, :rebel_1) | (:group, :rebel) | |
| (:group, :syrian) | (:Syrian, :group) | ||
| (:not, :support_1) | |||
| (:posing, :states) | (:pose_1, :united_states_1_States) | ||
| (:rebels, :support) | (:rebel_2, :support) | ||
| (:states, :support) | (:United_States, :support) | ||
| (:states, :support_1) | (:support_1, :united_states_1_States) | ||
| (:states, :want) | (:united_states_1_States, :want_1) | (:United_States, :want) | |
| (:support, :want) | (:support, :want) | ||
| (:support_1, :want) | (:support_1, :want_1) | ||
| * (:Effective, :uprising_1) | |||
| * (:Most, :uprising_1) | |||
| * (:approval_1, :stamp_1) | |||
| (:become_1, :uprising_1) | |||
| * (:but_not_1, :support_1) | |||
| (:united_states_1_States, :united_states_1_countries) | |||
| * (:Explicit, :stamp_1) | |||
| * (:group_1, :stamp_1) | |||
| * (<attr:about-7r>, :effective) | |||
| * (<entity:islamic>, :extremist) | |||
| * (:approval, :stamp) | |||
| * (:become, :one) | |||
| * (:effective, :force) | |||
| * (:effective, :group) | |||
| (:explicit, :group) | |||
| * (:explicit, :stamp) | |||
| (<attr:not-1r>, :extremist) | |||
| (:extremist, :support) | |||
| * (:force, :one) | |||
| * (:group, :stamp) | |||
| S2 | (:abroad, :donors) | (:Abroad, :donor_1) | (<attr:abroad-1r>, :donor) |
| (:donors, :flows) | (:donor_1, :flow_1) | ||
| (:donors, :group) | (:donor, :group) | ||
| (:donors, :like-minded) | (:Like-minded, :donor_1) | (:donor, :like-mind) | |
| (:donors, :money) | |||
| (:flows, :group) | (:flow_1, :group_1) | (:flow, :group) | |
| (:flows, :money) | (:flow_1, :money_1) | (<entity:money>, :flow) | |
| (:front, :group) | (:Nusra_Front, :group) | ||
| (:Nusra_front, :flow_1) | |||
| S3 | (:bases, :capture) | (:capture_1, :military_bases_1) | (:basis, :capture) |
| (:bases, :military) | (:basis, :military) | ||
| (:battalions, :capture) | (:battalion_1, :capture_1) | (:battalion, :capture) | |
| (:battalions, :lead) | (:Lead, :battalion_1) | (:battalion, :lead) | |
| (:battalions, :other) | (:battalion, :other) | ||
| (:boldness, :have) | (:boldness_1_boldness, :have_1) | (:boldness, :have) | |
| (:capture, :fields) | (:capture_1, :oil_fields_1) | (:capture, :field) | |
| (:capture, :lead) | (:capture, :lead) | ||
| (:fields, :oil) | (:field, :oil) | ||
| (:fighters, :have) | (:fighter_1, :have_1) | (:fighter, :have) | |
| (:fighters, :lead) | (:fighter, :lead) | ||
| (:fighters, :minority) | (:fighter_1, :minority_1) | ||
| (:fighters, :storm) | (:fighter, :storm) | ||
| (:fortified, :positions) | (:fortify_1, :storm_1_positions) | (:fortify, :position) | |
| (:have, :skill) | (:boldness_1_skill, :have_1) | (:have, :skill) | |
| (:minority, :rebels) | (:minority_1, :rebel_1) | (:minority, :rebel) | |
| (:minority, :small) | (:Small, :minority_1) | (:minority, :small) | |
| (:positions, :storm) | (:storm_1_positions, :storm_1_storm) | (:position, :storm) | |
| (:battalion_1, :boldness_1_skill) | |||
| (:boldness_1_boldness, :boldness_1_skill) | |||
| (:fighter_1, :neuter_1) | |||
| * (:boldness_1_skill, :storm_1_storm) | |||
| * (:skill, :storm) | |||
| S4 | (:attract, :fighters) | (:attract_1, :fighter_1) | (:attract, :fighter) |
| (:attract, :they) | (:attract_1, :thing_2) | (:attract, :they_2) | |
| (:gather, :they) | (:gather_1, :thing_2) | (:gather, :they_2) | |
| (:gather, :weapons) | (:gather_1, :weapon_1) | (:gather, :weapon) | |
| (:mount, :successes) | (:mount_1, :success_1) | (:mount, :success) | |
| (:successes, :their) | (:success_1, :thing_1) | (:success, :they) | |
| (:their, :they) | (:thing_1, :thing_2) | (:they, :they_2) | |
| * (:More_more_1, :More_more_2) | |||
| * (:More_more_1, :weapon_1) | |||
| * (:More_more_2, :fighter_1) | |||
| * (:fighter, :more_2) | |||
| * (:more, :weapon) | |||
| S5 | (:contributed, :fighters) | (:contribute_1, :fighter_1_fighters) | (:contribute, :fighter) |
| (:contributed, :say) | |||
| (:contributed, :weapons) | (:contribute_1, :fighter_1_weapons) | (:contribute, :weapon) | |
| (:contributed, :which) | |||
| (:direct, :offshoot) | (:direct, :offshoot) | ||
| (:fighters, :veteran) | (:Veteran, :fighter_1_fighters) | ||
| (:former, :insurgents) | (:Former, :iraqi_insurgents_1) | (:former, :insurgent) | |
| (:group, :is) | (:be, :group) | ||
| (:group, :offshoot) | (:group, :offshoot) | ||
| (:insurgents, :iraqi_2) | |||
| (:insurgents, :say) | (:iraqi_insurgents_1, :say_1) | (:insurgent, :say) | |
| (:iraqi_1, :officials) | (:Iraqi_Iraqi_1, :official_1) | (:Iraqi, :official) | |
| (:is, :offshoot) | (:be, :offshoot) | ||
| (:is, :say) | (:be, :say) | ||
| (:officials, :say) | (:official, :say) | ||
| (:offshoot, :qaeda) | (:Al_Qaeda, :offshoot) | ||
| (:qaeda, :which) | |||
| * (:Direct, :group_1) | |||
| (:fighter_1_fighters, :fighter_1_weapons) | |||
| * (:Al_qaeda_in_iraq, :group_1) | |||
| * (:Iraqi_Iraqi_1, :Iraqi_Iraqi_2) | |||
| (:Al_qaeda_in_iraq, :official_1) | |||
| * (:Al_Qaeda, :Iraq) | |||
| * (:fighter, :have) | |||
| (:fighter, :weapon) | |||
| (:be, :former) | |||
| * (:Al_Qaeda, :have) | |||
| * (:have, :weapon) | |||
| (:insurgent, :official) | |||
| S6 | (:brothers, :favor) | (:brother, :favor) | |
| (:brothers, :fought) | (:brother_1, :fight_1) | (:brother, :fight) | |
| (:brothers, :our) | (:brother_1, :person_1) | (:brother, :we) | |
| (:brothers, :returning) | (:brother_1, :return_1) | (:brother, :return) | |
| (:brothers, :syrian) | (:Syrian, :brother_1) | (:Syrian, :brother) | |
| (:efforts, :front) | (:Nusra_front, :effort_1) | (:Nusra_Front, :effort) | |
| (:efforts, :lead) | (:effort_1, :lead_1) | (:effort, :lead) | |
| (:efforts, :syria) | (:Syria_Syria, :effort_1) | (:Syria, :effort) | |
| (:favor, :returning) | (:favor_1, :return_1) | (:favor, :return) | |
| (:fought, :lands) | (:fight, :land) | ||
| (:fought, :us) | (:fight_1, :person_2) | (:fight, :we_2) | |
| (:he, :helped) | (:he, :help) | ||
| (:he, :lead) | (:Al_qaeda_in_iraq_he, :lead_1) | (:he, :lead) | |
| (:he, :veteran) | |||
| (:helped, :lead) | (:help, :lead) | ||
| (:helped, :said_2) | (:help, :say) | ||
| (:iraq_1, :iraq_2) | (:Iraq_Iraq_1, :Iraq_Iraq_2) | (:Iraq, :Iraq_2) | |
| (:iraq_1, :lands) | (:Iraq, :land) | ||
| (:is, :said_1) | |||
| (:is, :this) | |||
| (:is, :way) | |||
| (:our, :us) | (:person_1, :person_2) | (:we, :we_2) | |
| (:qaeda, :veteran) | (:Al_qaeda_in_iraq_Qaeda, :veteran_1) | (:Al_Qaeda, :veteran) | |
| (:returning, :way) | (:return, :way) | ||
| (:said_1, :veteran) | (:say_1, :veteran_1) | (:say_2, :veteran) | |
| (:said_2, :who) | |||
| (:simple, :way) | (<attr:simple-1a>, :way) | ||
| (:this, :way) | (:thing_1, :way_1) | ||
| (:veteran, :who) | |||
| * (:Just, :Simple) | |||
| * (:Iraq_Iraq_1, :fight_1) | |||
| (:Al_qaeda_in_iraq_Qaeda, :Al_qaeda_in_iraq_he) | |||
| * (:Al_qaeda_in_iraq_he, :say_2) | |||
| * (:lead_1, :say_2) | |||
| * (:return_1, :thing_1) | |||
| * (:Simple, :thing_1) | |||
| * (:Syria_Syria, :Syria_Syrian) | |||
| * (<attr:merely-1r>, :way) | |||
| (:effort, :he) | |||
| * (:Al_Qaeda, :Iraq_2) | |||
| (:Iraq_2, :say) | |||
| (:lead, :return) | |||
| (:Al_Qaeda, :say) | |||
| S7 | (:al-assad, :president) | (:Bashar_al-assad_al-Assad, :Syria_president), (:Bashar_al-assad_al-Assad, :Bashar_al-assad_president) | (:Bashar_al-Assad, :president) |
| (:al-assad, :running) | (:Bashar_al-Assad, :run) | ||
| (:group, :isolate) | (:group_1, :isolate_1) | (:group, :isolate) | |
| (:group, :it) | |||
| (:hopes, :isolate) | (:hope, :isolate) | ||
| (:hopes, :states) | (:hope_1, :united_states_1) | (:United_States, :hope) | |
| (:inheriting, :it) | (:inherit, :it) | ||
| (:inheriting, :prevent) | (:inherit, :prevent) | ||
| (:inheriting, :syria_2) | (:Syria_Syria_2, :inherit_1) | (:Syria_2, :inherit) | |
| (:isolate, :prevent) | |||
| (:isolate, :states) | (:isolate_1, :united_states_1) | ||
| (:it, :prevent) | (:prevent_1, :time_1_it) | (:it, :prevent) | |
| (:president, :syria_1) | (:Syria_Syria_1, :Syria_president) | (:Syria, :president) | |
| (:prevent, :states) | (:United_States, :prevent) | ||
| (:running, :sensing) | (:run, :sense) | ||
| (:running, :time) | (:run_1, :time_1_time) | (:run, :time) | |
| (:sensing, :states) | (:sense_1, :united_states_1) | (:United_States, :sense) | |
| (:syria_1, :syria_2) | (:Syria_Syria_1, :Syria_Syria_2) | (:Syria, :Syria_2) | |
| * (:Out, :run_1) | |||
| * (:be_1, :run_1) | |||
| * (:be_1, :sense_1) | |||
| * (:be_1, :time_1_time) | |||
| * (:group_1, :prevent_1) | |||
| (:time_1_it, :time_1_time) | |||
| (:Syria_president, :run_1) | |||
| (:Syria_Syria_2, :Syria_president) | |||
| * (:Bashar_al-Assad, :may) | |||
| * (:Bashar_al-Assad, :out) | |||
| (:United_States, :it) | |||
| * (:may, :run) | |||
| * (:may, :sense) | |||
| S8 | (:alternative, :government) | (:alternative, :government) | |
| (:americans, :dealings) | (:Americans, :dealings) | ||
| (:americans, :have) | (:Americans, :have_1) | (:Americans, :have) | |
| (:americans, :making) | (:Americans, :make_1) | ||
| (:blacklist, :front) | (:Nusra_front, :blacklist_1) | (:Nusra_Front, :blacklist) | |
| (:blacklist, :it_1) | (:blacklist, :it) | ||
| (:blacklist, :plans) | (:blacklist_1, :plan_1) | (:blacklist, :plan) | |
| (:dealings, :financial) | (:Financial, :dealings_1) | (<attr:fiscal-1a>, :dealings) | |
| (:dealings, :group) | (:dealings_1, :group_1) | (:dealings, :group) | |
| (:dealings, :have) | (:dealings_1, :have_1) | (:dealings, :have) | |
| (:europe, :prompting) | |||
| (:europe, :sanctions) | (:Europe, :sanction_1) | (:Europe, :sanction) | |
| (:front, :group) | |||
| (:government, :organize) | (:government_1, :organize_1) | (:government, :organize) | |
| (:government, :viable) | (:Viable, :government_1) | (<attr:feasible-1a>, :government) | |
| (:have, :making) | (:have, :make) | ||
| (:illegal, :making) | (:illegal, :make) | ||
| (:it_1, :making) | (:it, :make) | ||
| (:it_1, :plans) | (:it, :plan) | ||
| (:it_1, :prompting) | |||
| (:it_1, :states) | (:United_States, :it) | ||
| (:opposition, :organize) | (:organize_1, :syrian_opposition_1) | (:opposition, :organize) | |
| (:opposition, :pushes) | (:opposition, :push) | ||
| (:opposition, :syrian) | (<entity:syrian>, :opposition) | ||
| (:organization, :terrorist) | (:Terrorist, :organization_1) | ||
| (:organize, :pushes) | (:organize_1, :push_1) | ||
| (:prompting, :sanctions) | (:prompt_1, :sanction_1) | (:prompt, :sanction) | |
| (:pushes, :states) | (:United_states, :push_1) | (:United_States, :push) | |
| * (:Similar, :sanction_1) | |||
| * (:blacklist_1, :organization_1) | |||
| * (:United_states, :blacklist_1) | |||
| (:Illegal, :United_states) | |||
| * (:United_states, :make_1) | |||
| * (:United_states, :plan_1) | |||
| * (:United_states, :prompt_1) | |||
| * (:blacklist, :organization) | |||
| * (:have, :it_2) | |||
| * (:illegal, :it_2) | |||
| * (:it, :it_2) | |||
| * (:it_2, :make) | |||
| * (:United_States, :it_2) | |||
| (:United_States, :opposition) | |||
| * (:sanction, :similar) | |||
| Results w.r.t. gold standard | f1=0.677 p=0.869 r=0.555 tp=86 fp=13 fn=69 | f1=0.844 p=0.910 r=0.787 tp=122 fp=12 fn=33 | |
| Results w.r.t. union of correct answers | f1=0.732 p=0.869 r=0.632 tp=86 fp=13 fn=50 | f1=0.904 p=0.910 r=0.897 tp=122 fp=12 fn=14 | |