Commit 4a740d66 authored by Shyam Upadhyay's avatar Shyam Upadhyay
Browse files

can run eval on abstract using 80 HandAnn docs

parent 7da41e97
...@@ -26,6 +26,31 @@ public class Evaluator { ...@@ -26,6 +26,31 @@ public class Evaluator {
} }
/**
* to project hand annotated salience from abstract to body
* @param goldDoc
* @param goldAnn
*/
public static void projectGoldOnDoc(AnnotatedDocument goldDoc,List<EventInstance> goldAnn){
List<EventInstance> abs_events = goldDoc.abs_events;
List<EventInstance> body_events = goldDoc.body_events;
for (EventInstance ee : goldAnn) {
if(ee.salient==0)
continue;
for (EventInstance aa : abs_events) {
if (ee.pred_srl.getSurfaceForm().toLowerCase().equals(aa.pred_srl.getSurfaceForm().toLowerCase())) {
ee.salient = 1;
}
}
}
int docsalient = 0;
for (EventInstance ee : body_events) {
if (ee.salient == 1)
docsalient++;
}
}
static List<EventInstance> readGoldData(String goldFile) throws FileNotFoundException { static List<EventInstance> readGoldData(String goldFile) throws FileNotFoundException {
List<EventInstance> gold = new ArrayList<>(); List<EventInstance> gold = new ArrayList<>();
List<String> lines = LineIO.read(goldFile); List<String> lines = LineIO.read(goldFile);
...@@ -55,15 +80,19 @@ public class Evaluator { ...@@ -55,15 +80,19 @@ public class Evaluator {
for(EventInstance inst:predevents) for(EventInstance inst:predevents)
{ {
// todo make this based on salience // todo make this based on salience
r.incrementPredicted(); if(inst.salient==1)
r.incrementPredicted();
} }
for(EventInstance gold:goldann) for(EventInstance gold:goldann)
r.incrementGold(); {
if(gold.salient==1)
r.incrementGold();
}
for(EventInstance inst:predevents) for(EventInstance inst:predevents)
{ {
for(EventInstance gold:goldann) for(EventInstance gold:goldann)
{ {
if(gold.start==inst.start && gold.end==inst.end) if(gold.start==inst.start && gold.end==inst.end && gold.salient==inst.salient && gold.salient==1)
{ {
System.out.println(inst); System.out.println(inst);
r.incrementCorrect(); r.incrementCorrect();
...@@ -114,9 +143,9 @@ public class Evaluator { ...@@ -114,9 +143,9 @@ public class Evaluator {
static int emptyAbstracts = 0; static int emptyAbstracts = 0;
public static void EvaluateOnAbstractFrames(List<EventInstance> absEvents, public static void EvaluateOnAbstractFrames(List<EventInstance> absEvents,
List<EventInstance> bodyEvents, ClassificationTester tester, List<EventInstance> bodyEvents, ClassificationTester tester,
List<String> acceptableFrames) { List<String> acceptableFrames) {
List<String>triggered_frame_surfaces = new ArrayList<String>(); List<String>triggered_frame_surfaces = new ArrayList<String>();
for(EventInstance ee:bodyEvents) for(EventInstance ee:bodyEvents)
{ {
...@@ -163,6 +192,6 @@ public class Evaluator { ...@@ -163,6 +192,6 @@ public class Evaluator {
tester.record("ABSENT","PRESENT"); tester.record("ABSENT","PRESENT");
} }
} }
} }
} }
...@@ -69,22 +69,26 @@ public class ForBrat { ...@@ -69,22 +69,26 @@ public class ForBrat {
// Annotation data = reader.parse("/shared/corpora/corporaWeb/written/multi/ACE-2005/Extracted/data/English/bc/fp1/CNN_CF_20030303.1900.06-2.apf.xml"); // Annotation data = reader.parse("/shared/corpora/corporaWeb/written/multi/ACE-2005/Extracted/data/English/bc/fp1/CNN_CF_20030303.1900.06-2.apf.xml");
} }
public static void writeEvents(String docid, List<EventInstance> abs_events, TextAnnotation absTa, String suffix) throws FileNotFoundException { /**
* suffix is "abs" or "body"
* @param docid
* @param events
* @param ta
* @param suffix
* @throws FileNotFoundException
*/
public static void writeEvents(String docid, List<EventInstance> events, TextAnnotation ta, String suffix) throws FileNotFoundException {
String path = Params.brat_data + "/"+docid+"_"+suffix; String path = Params.brat_data + "/"+docid+"_"+suffix;
PrintWriter w = new PrintWriter(path+".txt"); PrintWriter w = new PrintWriter(path+".txt");
String text = absTa.getText(); String text = ta.getText();
w.println(text); w.println(text);
w.close(); w.close();
PrintWriter ann = new PrintWriter(path+".ann"); PrintWriter ann = new PrintWriter(path+".ann");
for(int i=0;i<abs_events.size();i++) for(int i=0;i<events.size();i++)
{ {
EventInstance ee = abs_events.get(i); EventInstance ee = events.get(i);
String buf=""; String buf="";
// for(Frame ff:ee.frames)
// {
// buf+=ff.getName()+"--";
// }
buf+=ee.frame+"--"; buf+=ee.frame+"--";
int start = ee.pred_srl.getStartCharOffset(); int start = ee.pred_srl.getStartCharOffset();
int end = ee.pred_srl.getEndCharOffset(); int end = ee.pred_srl.getEndCharOffset();
......
...@@ -5,9 +5,11 @@ import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation ...@@ -5,9 +5,11 @@ import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation
import edu.illinois.cs.cogcomp.core.experiments.EvaluationRecord; import edu.illinois.cs.cogcomp.core.experiments.EvaluationRecord;
import edu.illinois.cs.cogcomp.core.io.IOUtils; import edu.illinois.cs.cogcomp.core.io.IOUtils;
import edu.illinois.cs.cogcomp.core.io.LineIO; import edu.illinois.cs.cogcomp.core.io.LineIO;
import edu.illinois.cs.cogcomp.edison.utilities.EdisonException;
import edu.illinois.cs.cogcomp.nytlabs.corpus.FramenetExtractor; import edu.illinois.cs.cogcomp.nytlabs.corpus.FramenetExtractor;
import edu.illinois.cs.cogcomp.nytlabs.corpus.MyCuratorClient; import edu.illinois.cs.cogcomp.nytlabs.corpus.MyCuratorClient;
import edu.illinois.cs.cogcomp.nytlabs.corpus.Params; import edu.illinois.cs.cogcomp.nytlabs.corpus.Params;
import edu.illinois.cs.cogcomp.utils.TAUtils;
import sun.nio.ch.IOUtil; import sun.nio.ch.IOUtil;
import java.io.File; import java.io.File;
...@@ -75,7 +77,7 @@ public class LazyNYTReader{ ...@@ -75,7 +77,7 @@ public class LazyNYTReader{
List<EventInstance> body_events = extractor.extract(bodyTa); List<EventInstance> body_events = extractor.extract(bodyTa);
doc = new AnnotatedDocument(body_events, abs_events); doc = new AnnotatedDocument(body_events, abs_events);
doc.nydoc = nydoc; doc.nydoc = nydoc;
// doc.bestpara = extractGoldParagraphs(nydoc, cc); // doc.bestpara = extractGoldParagraphs(nydoc, cc);
return doc; return doc;
} }
...@@ -187,6 +189,7 @@ public class LazyNYTReader{ ...@@ -187,6 +189,7 @@ public class LazyNYTReader{
while(doc!=null) while(doc!=null)
{ {
System.out.println(doc.nydoc.getGuid()); System.out.println(doc.nydoc.getGuid());
ApplyHeuristic(doc);
Evaluator.evaluateOnGoldData(r, Params.goldPath + "/" + doc.nydoc.getGuid() +"_abs.ann",doc.abs_events); Evaluator.evaluateOnGoldData(r, Params.goldPath + "/" + doc.nydoc.getGuid() +"_abs.ann",doc.abs_events);
doc=reader.next(); doc=reader.next();
} }
...@@ -196,6 +199,26 @@ public class LazyNYTReader{ ...@@ -196,6 +199,26 @@ public class LazyNYTReader{
System.out.println(r.getCorrectCount()+" "+r.getPredictedCount()+" "+r.getGoldCount()+" "+r.getPrecision()+" "+r.getRecall()+" "+r.getF1()); System.out.println(r.getCorrectCount()+" "+r.getPredictedCount()+" "+r.getGoldCount()+" "+r.getPrecision()+" "+r.getRecall()+" "+r.getF1());
System.exit(-1); System.exit(-1);
} }
System.out.println(r.getPrecision()+" "+r.getRecall()+" "+r.getF1()); System.out.println("Gold:"+r.getGoldCount()+" P:"+r.getPrecision()+" R:"+r.getRecall()+" F:"+r.getF1());
}
private static void ApplyHeuristic(AnnotatedDocument doc) throws EdisonException {
for(EventInstance ee:doc.abs_events)
{
// System.out.println(ee.pred_srl+" "+ TAUtils.getClauseNature(ee.pred_srl));
System.out.println(ee.pred_srl+" "+ TAUtils.voiceFeat(ee.pred_srl).getName());
// if(TAUtils.getClauseNature(ee.pred_srl).equals("main"))
// {
// ee.salient=1;
// }
// if(TAUtils.getClauseNature(ee.pred_srl).equals("relative"))
// {
// ee.salient=1;
// }
if(TAUtils.voiceFeat(ee.pred_srl).getName().equals("A"))
{
ee.salient=1;
}
}
} }
} }
...@@ -61,16 +61,6 @@ public class Main { ...@@ -61,16 +61,6 @@ public class Main {
// System.out.println(para0); // System.out.println(para0);
// } // }
private static void TryHeuristic(AnnotatedDocument doc) {
System.out.println("Trying ...");
// System.out.println(doc.absTa.text);
// System.out.println(doc.nydoc.getArticleAbstract());
System.out.println("PARA0:"+doc.nydoc.paragraphs.get(0));
for(EventInstance ee:doc.body_events)
{
System.out.println(ee.pred_srl+" "+TAUtils.getClauseNature(ee.pred_srl));
}
}
// @CommandDescription(description = "String numDocs,String alreadyCached") // @CommandDescription(description = "String numDocs,String alreadyCached")
// public static void NoisySupervisionTrain(String numDocs,String alreadyCached) throws Exception { // public static void NoisySupervisionTrain(String numDocs,String alreadyCached) throws Exception {
......
...@@ -55,12 +55,12 @@ public class EventBasicFeatureExtractor { ...@@ -55,12 +55,12 @@ public class EventBasicFeatureExtractor {
private void addClauseNatureVoiceConjFeats(EventInstance m, Map<String, Float> fmap) { private void addClauseNatureVoiceConjFeats(EventInstance m, Map<String, Float> fmap) {
String nature=TAUtils.getClauseNature(m.pred_srl); String nature=TAUtils.getClauseNature(m.pred_srl);
fmap.put("clause_"+nature, 1.0f); fmap.put("clause_"+nature, 1.0f);
try { // try {
Set<Feature> voice = TAUtils.voiceFeat(m.pred_srl); // Set<Feature> voice = TAUtils.voiceFeat(m.pred_srl);
fmap.put("voice_"+voice.toString(),10.f); // fmap.put("voice_"+voice.toString(),10.f);
} catch (EdisonException e) { // } catch (EdisonException e) {
e.printStackTrace(); // e.printStackTrace();
} // }
} }
......
...@@ -235,8 +235,14 @@ public class TAUtils { ...@@ -235,8 +235,14 @@ public class TAUtils {
return "main"; return "main";
} }
public static Set<Feature> voiceFeat(Constituent verb) throws EdisonException { /**
return voice.getFeatures(verb); * returns [A],[P] or [X]
* @param verb
* @return
* @throws EdisonException
*/
public static Feature voiceFeat(Constituent verb) throws EdisonException {
return voice.getFeatures(verb).toArray(new Feature[1])[0];
} }
public static VerbVoiceIndicator voice = new VerbVoiceIndicator(ViewNames.PARSE_STANFORD); public static VerbVoiceIndicator voice = new VerbVoiceIndicator(ViewNames.PARSE_STANFORD);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment