Skip to content
Snippets Groups Projects
Commit 4a740d66 authored by Shyam Upadhyay's avatar Shyam Upadhyay
Browse files

can run eval on abstract using 80 HandAnn docs

parent 7da41e97
No related branches found
No related tags found
No related merge requests found
......@@ -26,6 +26,31 @@ public class Evaluator {
}
/**
* to project hand annotated salience from abstract to body
* @param goldDoc
* @param goldAnn
*/
public static void projectGoldOnDoc(AnnotatedDocument goldDoc,List<EventInstance> goldAnn){
List<EventInstance> abs_events = goldDoc.abs_events;
List<EventInstance> body_events = goldDoc.body_events;
for (EventInstance ee : goldAnn) {
if(ee.salient==0)
continue;
for (EventInstance aa : abs_events) {
if (ee.pred_srl.getSurfaceForm().toLowerCase().equals(aa.pred_srl.getSurfaceForm().toLowerCase())) {
ee.salient = 1;
}
}
}
int docsalient = 0;
for (EventInstance ee : body_events) {
if (ee.salient == 1)
docsalient++;
}
}
static List<EventInstance> readGoldData(String goldFile) throws FileNotFoundException {
List<EventInstance> gold = new ArrayList<>();
List<String> lines = LineIO.read(goldFile);
......@@ -55,15 +80,19 @@ public class Evaluator {
for(EventInstance inst:predevents)
{
// todo make this based on salience
r.incrementPredicted();
if(inst.salient==1)
r.incrementPredicted();
}
for(EventInstance gold:goldann)
r.incrementGold();
{
if(gold.salient==1)
r.incrementGold();
}
for(EventInstance inst:predevents)
{
for(EventInstance gold:goldann)
{
if(gold.start==inst.start && gold.end==inst.end)
if(gold.start==inst.start && gold.end==inst.end && gold.salient==inst.salient && gold.salient==1)
{
System.out.println(inst);
r.incrementCorrect();
......@@ -114,9 +143,9 @@ public class Evaluator {
static int emptyAbstracts = 0;
public static void EvaluateOnAbstractFrames(List<EventInstance> absEvents,
List<EventInstance> bodyEvents, ClassificationTester tester,
List<String> acceptableFrames) {
public static void EvaluateOnAbstractFrames(List<EventInstance> absEvents,
List<EventInstance> bodyEvents, ClassificationTester tester,
List<String> acceptableFrames) {
List<String>triggered_frame_surfaces = new ArrayList<String>();
for(EventInstance ee:bodyEvents)
{
......@@ -163,6 +192,6 @@ public class Evaluator {
tester.record("ABSENT","PRESENT");
}
}
}
}
}
......@@ -69,22 +69,26 @@ public class ForBrat {
// Annotation data = reader.parse("/shared/corpora/corporaWeb/written/multi/ACE-2005/Extracted/data/English/bc/fp1/CNN_CF_20030303.1900.06-2.apf.xml");
}
public static void writeEvents(String docid, List<EventInstance> abs_events, TextAnnotation absTa, String suffix) throws FileNotFoundException {
/**
* suffix is "abs" or "body"
* @param docid
* @param events
* @param ta
* @param suffix
* @throws FileNotFoundException
*/
public static void writeEvents(String docid, List<EventInstance> events, TextAnnotation ta, String suffix) throws FileNotFoundException {
String path = Params.brat_data + "/"+docid+"_"+suffix;
PrintWriter w = new PrintWriter(path+".txt");
String text = absTa.getText();
String text = ta.getText();
w.println(text);
w.close();
PrintWriter ann = new PrintWriter(path+".ann");
for(int i=0;i<abs_events.size();i++)
for(int i=0;i<events.size();i++)
{
EventInstance ee = abs_events.get(i);
EventInstance ee = events.get(i);
String buf="";
// for(Frame ff:ee.frames)
// {
// buf+=ff.getName()+"--";
// }
buf+=ee.frame+"--";
int start = ee.pred_srl.getStartCharOffset();
int end = ee.pred_srl.getEndCharOffset();
......
......@@ -5,9 +5,11 @@ import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation
import edu.illinois.cs.cogcomp.core.experiments.EvaluationRecord;
import edu.illinois.cs.cogcomp.core.io.IOUtils;
import edu.illinois.cs.cogcomp.core.io.LineIO;
import edu.illinois.cs.cogcomp.edison.utilities.EdisonException;
import edu.illinois.cs.cogcomp.nytlabs.corpus.FramenetExtractor;
import edu.illinois.cs.cogcomp.nytlabs.corpus.MyCuratorClient;
import edu.illinois.cs.cogcomp.nytlabs.corpus.Params;
import edu.illinois.cs.cogcomp.utils.TAUtils;
import sun.nio.ch.IOUtil;
import java.io.File;
......@@ -75,7 +77,7 @@ public class LazyNYTReader{
List<EventInstance> body_events = extractor.extract(bodyTa);
doc = new AnnotatedDocument(body_events, abs_events);
doc.nydoc = nydoc;
// doc.bestpara = extractGoldParagraphs(nydoc, cc);
// doc.bestpara = extractGoldParagraphs(nydoc, cc);
return doc;
}
......@@ -187,6 +189,7 @@ public class LazyNYTReader{
while(doc!=null)
{
System.out.println(doc.nydoc.getGuid());
ApplyHeuristic(doc);
Evaluator.evaluateOnGoldData(r, Params.goldPath + "/" + doc.nydoc.getGuid() +"_abs.ann",doc.abs_events);
doc=reader.next();
}
......@@ -196,6 +199,26 @@ public class LazyNYTReader{
System.out.println(r.getCorrectCount()+" "+r.getPredictedCount()+" "+r.getGoldCount()+" "+r.getPrecision()+" "+r.getRecall()+" "+r.getF1());
System.exit(-1);
}
System.out.println(r.getPrecision()+" "+r.getRecall()+" "+r.getF1());
System.out.println("Gold:"+r.getGoldCount()+" P:"+r.getPrecision()+" R:"+r.getRecall()+" F:"+r.getF1());
}
private static void ApplyHeuristic(AnnotatedDocument doc) throws EdisonException {
for(EventInstance ee:doc.abs_events)
{
// System.out.println(ee.pred_srl+" "+ TAUtils.getClauseNature(ee.pred_srl));
System.out.println(ee.pred_srl+" "+ TAUtils.voiceFeat(ee.pred_srl).getName());
// if(TAUtils.getClauseNature(ee.pred_srl).equals("main"))
// {
// ee.salient=1;
// }
// if(TAUtils.getClauseNature(ee.pred_srl).equals("relative"))
// {
// ee.salient=1;
// }
if(TAUtils.voiceFeat(ee.pred_srl).getName().equals("A"))
{
ee.salient=1;
}
}
}
}
......@@ -61,16 +61,6 @@ public class Main {
// System.out.println(para0);
// }
private static void TryHeuristic(AnnotatedDocument doc) {
System.out.println("Trying ...");
// System.out.println(doc.absTa.text);
// System.out.println(doc.nydoc.getArticleAbstract());
System.out.println("PARA0:"+doc.nydoc.paragraphs.get(0));
for(EventInstance ee:doc.body_events)
{
System.out.println(ee.pred_srl+" "+TAUtils.getClauseNature(ee.pred_srl));
}
}
// @CommandDescription(description = "String numDocs,String alreadyCached")
// public static void NoisySupervisionTrain(String numDocs,String alreadyCached) throws Exception {
......
......@@ -55,12 +55,12 @@ public class EventBasicFeatureExtractor {
private void addClauseNatureVoiceConjFeats(EventInstance m, Map<String, Float> fmap) {
String nature=TAUtils.getClauseNature(m.pred_srl);
fmap.put("clause_"+nature, 1.0f);
try {
Set<Feature> voice = TAUtils.voiceFeat(m.pred_srl);
fmap.put("voice_"+voice.toString(),10.f);
} catch (EdisonException e) {
e.printStackTrace();
}
// try {
// Set<Feature> voice = TAUtils.voiceFeat(m.pred_srl);
// fmap.put("voice_"+voice.toString(),10.f);
// } catch (EdisonException e) {
// e.printStackTrace();
// }
}
......
......@@ -235,8 +235,14 @@ public class TAUtils {
return "main";
}
public static Set<Feature> voiceFeat(Constituent verb) throws EdisonException {
return voice.getFeatures(verb);
/**
* returns [A],[P] or [X]
* @param verb
* @return
* @throws EdisonException
*/
public static Feature voiceFeat(Constituent verb) throws EdisonException {
return voice.getFeatures(verb).toArray(new Feature[1])[0];
}
public static VerbVoiceIndicator voice = new VerbVoiceIndicator(ViewNames.PARSE_STANFORD);
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment