Commit 4a740d66 authored by Shyam Upadhyay's avatar Shyam Upadhyay
Browse files

can run eval on abstract using 80 HandAnn docs

parent 7da41e97
......@@ -26,6 +26,31 @@ public class Evaluator {
}
/**
* to project hand annotated salience from abstract to body
* @param goldDoc
* @param goldAnn
*/
public static void projectGoldOnDoc(AnnotatedDocument goldDoc,List<EventInstance> goldAnn){
List<EventInstance> abs_events = goldDoc.abs_events;
List<EventInstance> body_events = goldDoc.body_events;
for (EventInstance ee : goldAnn) {
if(ee.salient==0)
continue;
for (EventInstance aa : abs_events) {
if (ee.pred_srl.getSurfaceForm().toLowerCase().equals(aa.pred_srl.getSurfaceForm().toLowerCase())) {
ee.salient = 1;
}
}
}
int docsalient = 0;
for (EventInstance ee : body_events) {
if (ee.salient == 1)
docsalient++;
}
}
static List<EventInstance> readGoldData(String goldFile) throws FileNotFoundException {
List<EventInstance> gold = new ArrayList<>();
List<String> lines = LineIO.read(goldFile);
......@@ -55,15 +80,19 @@ public class Evaluator {
for(EventInstance inst:predevents)
{
// todo make this based on salience
if(inst.salient==1)
r.incrementPredicted();
}
for(EventInstance gold:goldann)
{
if(gold.salient==1)
r.incrementGold();
}
for(EventInstance inst:predevents)
{
for(EventInstance gold:goldann)
{
if(gold.start==inst.start && gold.end==inst.end)
if(gold.start==inst.start && gold.end==inst.end && gold.salient==inst.salient && gold.salient==1)
{
System.out.println(inst);
r.incrementCorrect();
......
......@@ -69,22 +69,26 @@ public class ForBrat {
// Annotation data = reader.parse("/shared/corpora/corporaWeb/written/multi/ACE-2005/Extracted/data/English/bc/fp1/CNN_CF_20030303.1900.06-2.apf.xml");
}
public static void writeEvents(String docid, List<EventInstance> abs_events, TextAnnotation absTa, String suffix) throws FileNotFoundException {
/**
* suffix is "abs" or "body"
* @param docid
* @param events
* @param ta
* @param suffix
* @throws FileNotFoundException
*/
public static void writeEvents(String docid, List<EventInstance> events, TextAnnotation ta, String suffix) throws FileNotFoundException {
String path = Params.brat_data + "/"+docid+"_"+suffix;
PrintWriter w = new PrintWriter(path+".txt");
String text = absTa.getText();
String text = ta.getText();
w.println(text);
w.close();
PrintWriter ann = new PrintWriter(path+".ann");
for(int i=0;i<abs_events.size();i++)
for(int i=0;i<events.size();i++)
{
EventInstance ee = abs_events.get(i);
EventInstance ee = events.get(i);
String buf="";
// for(Frame ff:ee.frames)
// {
// buf+=ff.getName()+"--";
// }
buf+=ee.frame+"--";
int start = ee.pred_srl.getStartCharOffset();
int end = ee.pred_srl.getEndCharOffset();
......
......@@ -5,9 +5,11 @@ import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation
import edu.illinois.cs.cogcomp.core.experiments.EvaluationRecord;
import edu.illinois.cs.cogcomp.core.io.IOUtils;
import edu.illinois.cs.cogcomp.core.io.LineIO;
import edu.illinois.cs.cogcomp.edison.utilities.EdisonException;
import edu.illinois.cs.cogcomp.nytlabs.corpus.FramenetExtractor;
import edu.illinois.cs.cogcomp.nytlabs.corpus.MyCuratorClient;
import edu.illinois.cs.cogcomp.nytlabs.corpus.Params;
import edu.illinois.cs.cogcomp.utils.TAUtils;
import sun.nio.ch.IOUtil;
import java.io.File;
......@@ -187,6 +189,7 @@ public class LazyNYTReader{
while(doc!=null)
{
System.out.println(doc.nydoc.getGuid());
ApplyHeuristic(doc);
Evaluator.evaluateOnGoldData(r, Params.goldPath + "/" + doc.nydoc.getGuid() +"_abs.ann",doc.abs_events);
doc=reader.next();
}
......@@ -196,6 +199,26 @@ public class LazyNYTReader{
System.out.println(r.getCorrectCount()+" "+r.getPredictedCount()+" "+r.getGoldCount()+" "+r.getPrecision()+" "+r.getRecall()+" "+r.getF1());
System.exit(-1);
}
System.out.println(r.getPrecision()+" "+r.getRecall()+" "+r.getF1());
System.out.println("Gold:"+r.getGoldCount()+" P:"+r.getPrecision()+" R:"+r.getRecall()+" F:"+r.getF1());
}
private static void ApplyHeuristic(AnnotatedDocument doc) throws EdisonException {
for(EventInstance ee:doc.abs_events)
{
// System.out.println(ee.pred_srl+" "+ TAUtils.getClauseNature(ee.pred_srl));
System.out.println(ee.pred_srl+" "+ TAUtils.voiceFeat(ee.pred_srl).getName());
// if(TAUtils.getClauseNature(ee.pred_srl).equals("main"))
// {
// ee.salient=1;
// }
// if(TAUtils.getClauseNature(ee.pred_srl).equals("relative"))
// {
// ee.salient=1;
// }
if(TAUtils.voiceFeat(ee.pred_srl).getName().equals("A"))
{
ee.salient=1;
}
}
}
}
......@@ -61,16 +61,6 @@ public class Main {
// System.out.println(para0);
// }
private static void TryHeuristic(AnnotatedDocument doc) {
System.out.println("Trying ...");
// System.out.println(doc.absTa.text);
// System.out.println(doc.nydoc.getArticleAbstract());
System.out.println("PARA0:"+doc.nydoc.paragraphs.get(0));
for(EventInstance ee:doc.body_events)
{
System.out.println(ee.pred_srl+" "+TAUtils.getClauseNature(ee.pred_srl));
}
}
// @CommandDescription(description = "String numDocs,String alreadyCached")
// public static void NoisySupervisionTrain(String numDocs,String alreadyCached) throws Exception {
......
......@@ -55,12 +55,12 @@ public class EventBasicFeatureExtractor {
private void addClauseNatureVoiceConjFeats(EventInstance m, Map<String, Float> fmap) {
String nature=TAUtils.getClauseNature(m.pred_srl);
fmap.put("clause_"+nature, 1.0f);
try {
Set<Feature> voice = TAUtils.voiceFeat(m.pred_srl);
fmap.put("voice_"+voice.toString(),10.f);
} catch (EdisonException e) {
e.printStackTrace();
}
// try {
// Set<Feature> voice = TAUtils.voiceFeat(m.pred_srl);
// fmap.put("voice_"+voice.toString(),10.f);
// } catch (EdisonException e) {
// e.printStackTrace();
// }
}
......
......@@ -235,8 +235,14 @@ public class TAUtils {
return "main";
}
public static Set<Feature> voiceFeat(Constituent verb) throws EdisonException {
return voice.getFeatures(verb);
/**
* returns [A],[P] or [X]
* @param verb
* @return
* @throws EdisonException
*/
public static Feature voiceFeat(Constituent verb) throws EdisonException {
return voice.getFeatures(verb).toArray(new Feature[1])[0];
}
public static VerbVoiceIndicator voice = new VerbVoiceIndicator(ViewNames.PARSE_STANFORD);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment