UNLOZE/projects-jenz
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fixing pre discord launch calcaulations, implementation of JMWE coreNLP extension, hopefully fixed repetetive data in multithreading

Browse Source
This commit is contained in:
jenzur
2019-03-15 01:32:06 +01:00
parent 43da2dd5d5
commit 232190d076
3 changed files with 217 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java
+158 -1
View File
@@ -4,11 +4,15 @@ import FunctionLayer.LevenshteinDistance;
import FunctionLayer.MYSQLDatahandler;
import FunctionLayer.SimilarityMatrix;
import com.google.common.collect.MapMaker;
import edu.mit.jmwe.data.IMWE;
import edu.mit.jmwe.data.IMWEDesc;
import edu.mit.jmwe.data.IToken;
import edu.stanford.nlp.ie.AbstractSequenceClassifier;
import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.ling.HasWord;
import edu.stanford.nlp.ling.IndexedWord;
import edu.stanford.nlp.ling.JMWEAnnotation;
import edu.stanford.nlp.ling.TaggedWord;
import edu.stanford.nlp.neural.rnn.RNNCoreAnnotations;
import edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser;
@@ -48,14 +52,16 @@ import org.ejml.simple.SimpleMatrix;
*/
public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
private SimilarityMatrix smxParam;
private String str;
private String str1;
private SimilarityMatrix smxParam;
private ShiftReduceParser model;
private MaxentTagger tagger;
private GrammaticalStructureFactory gsf;
private StanfordCoreNLP pipeline;
private StanfordCoreNLP pipelineSentiment;
private StanfordCoreNLP pipelineJMWE;
private AbstractSequenceClassifier classifier;
public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam) {
@@ -66,6 +72,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
tagger = MYSQLDatahandler.getTagger();
pipeline = MYSQLDatahandler.getPipeline();
pipelineSentiment = MYSQLDatahandler.getPipelineSentiment();
pipelineJMWE = MYSQLDatahandler.getPipelineJMWE();
gsf = MYSQLDatahandler.getGsf();
classifier = MYSQLDatahandler.getClassifier();
}
@@ -231,6 +238,7 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
String partText = sentence.toString();
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
if (partText.length() > longest1) {
mainSentiment1 = sentiment;
longest1 = partText.length();
@@ -258,6 +266,155 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
score -= (deffLongest - deffshorter) * 50;
}
}
Annotation jmweStrAnnotation = new Annotation(str);
pipelineJMWE.annotate(jmweStrAnnotation);
List<CoreMap> sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class);
int tokensCounter1 = 0;
int tokensCounter2 = 0;
int anotatorcounter1 = 0;
int anotatorcounter2 = 0;
int inflectedCounterPositive1 = 0;
int inflectedCounterPositive2 = 0;
int inflectedCounterNegative = 0;
int MarkedContinuousCounter1 = 0;
int MarkedContinuousCounter2 = 0;
int UnmarkedPatternCounter = 0;
ConcurrentMap<Integer, String> ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strTokenStems2 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strTokenForm1 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strTokenForm2 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strTokenGetEntry1 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strTokenGetEntry2 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strTokenGetiPart1 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strTokenGetiPart2 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strTokenEntryPOS1 = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strTokenEntryPOS2 = new MapMaker().concurrencyLevel(2).makeMap();
for (CoreMap sentence : sentences) {
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
if (token.isInflected()) {
inflectedCounterPositive1++;
} else {
inflectedCounterNegative++;
}
strTokenForm1.put(strTokenForm1.size() + 1, token.getForm());
strTokenGetEntry1.put(strTokenGetEntry1.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
IMWEDesc entry = token.getEntry();
MarkedContinuousCounter1 += entry.getMarkedContinuous();
UnmarkedPatternCounter += entry.getUnmarkedPattern();
for (IMWEDesc.IPart iPart : values) {
strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm());
}
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
strTokenEntryPOS1.put(strTokenEntryPOS1.size() + 1, strPostPrefix);
}
for (IToken tokens : token.getTokens()) {
ITokenMapTag1.put(ITokenMapTag1.size() + 1, tokens.getTag());
for (String strtoken : tokens.getStems()) {
strTokenStems1.put(strTokenStems1.size() + 1, strtoken);
}
}
tokensCounter1++;
}
anotatorcounter1++;
}
jmweStrAnnotation = new Annotation(str1);
pipelineJMWE.annotate(jmweStrAnnotation);
sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class);
for (CoreMap sentence : sentences) {
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
if (token.isInflected()) {
inflectedCounterPositive2++;
} else {
inflectedCounterNegative--;
}
strTokenForm2.put(strTokenForm2.size() + 1, token.getForm());
strTokenGetEntry2.put(strTokenGetEntry2.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
IMWEDesc entry = token.getEntry();
MarkedContinuousCounter2 += entry.getMarkedContinuous();
UnmarkedPatternCounter += entry.getUnmarkedPattern();
for (IMWEDesc.IPart iPart : values) {
strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm());
}
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
strTokenEntryPOS2.put(strTokenEntryPOS2.size() + 1, strPostPrefix);
}
for (IToken tokens : token.getTokens()) {
ITokenMapTag2.put(ITokenMapTag2.size() + 1, tokens.getTag());
for (String strtoken : tokens.getStems()) {
strTokenStems2.put(strTokenStems2.size() + 1, strtoken);
}
}
tokensCounter2++;
}
anotatorcounter2++;
}
for (String strTokenPos1 : strTokenEntryPOS1.values()) {
for (String strTokenPos2 : strTokenEntryPOS2.values()) {
if (strTokenPos1.equals(strTokenPos2)) {
score += 500;
}
}
}
score += UnmarkedPatternCounter * 1600;
if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) {
score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500
: (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500;
}
for (String strTokeniPart1 : strTokenGetiPart1.values()) {
for (String strTokeniPart2 : strTokenGetiPart2.values()) {
if (strTokeniPart1.equals(strTokeniPart2)) {
score += 400;
}
}
}
for (String strTokenEntry1 : strTokenGetEntry1.values()) {
for (String strTokenEntry2 : strTokenGetEntry2.values()) {
if (strTokenEntry1.equals(strTokenEntry2)) {
score += 2500;
}
}
}
for (String strmapTag : ITokenMapTag1.values()) {
for (String strmapTag1 : ITokenMapTag2.values()) {
if (strmapTag.equals(strmapTag1)) {
score += 1450;
}
}
}
for (String strTokenForm1itr1 : strTokenForm1.values()) {
for (String strTokenForm1itr2 : strTokenForm2.values()) {
if (strTokenForm1itr1.equals(strTokenForm1itr2)) {
score += 2600;
} else if (strTokenForm1itr1.contains(strTokenForm1itr2)) {
score += 500;
}
}
}
for (String strTokenStem : strTokenStems1.values()) {
for (String strTokenStem1 : strTokenStems2.values()) {
if (strTokenStem.equals(strTokenStem1)) {
score += 1500;
}
}
}
if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) {
score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650;
}
if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) {
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
}
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
score += (anotatorcounter1 + anotatorcounter2) * 400;
}
if (tokensCounter1 > 0 && tokensCounter2 > 0) {
score += (tokensCounter1 + tokensCounter2) * 400;
} else {
score -= tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
}
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
int SentenceScoreDiff = leven.computeLevenshteinDistance();
SentenceScoreDiff *= 15;