package FunctionLayer.StanfordParser; import FunctionLayer.LevenshteinDistance; import FunctionLayer.MYSQLDatahandler; import FunctionLayer.SimilarityMatrix; import com.google.common.collect.MapMaker; import edu.mit.jmwe.data.IMWE; import edu.mit.jmwe.data.IMWEDesc; import edu.mit.jmwe.data.IToken; import edu.stanford.nlp.ie.AbstractSequenceClassifier; import edu.stanford.nlp.ling.CoreAnnotations; import edu.stanford.nlp.ling.CoreLabel; import edu.stanford.nlp.ling.HasWord; import edu.stanford.nlp.ling.IndexedWord; import edu.stanford.nlp.ling.JMWEAnnotation; import edu.stanford.nlp.ling.TaggedWord; import edu.stanford.nlp.neural.rnn.RNNCoreAnnotations; import edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser; import edu.stanford.nlp.pipeline.Annotation; import edu.stanford.nlp.pipeline.StanfordCoreNLP; import edu.stanford.nlp.process.DocumentPreprocessor; import edu.stanford.nlp.sentiment.SentimentCoreAnnotations; import edu.stanford.nlp.sequences.DocumentReaderAndWriter; import edu.stanford.nlp.tagger.maxent.MaxentTagger; import edu.stanford.nlp.trees.Constituent; import edu.stanford.nlp.trees.GrammaticalRelation; import edu.stanford.nlp.trees.GrammaticalStructure; import edu.stanford.nlp.trees.GrammaticalStructureFactory; import edu.stanford.nlp.trees.Tree; import edu.stanford.nlp.trees.TreeCoreAnnotations; import edu.stanford.nlp.trees.TypedDependency; import edu.stanford.nlp.trees.tregex.gui.Tdiff; import edu.stanford.nlp.util.CoreMap; import java.io.StringReader; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; import org.ejml.simple.SimpleMatrix; /* * To change this license header, choose License Headers in Project Properties. * To change this template file, choose Tools | Templates * and open the template in the editor. */ /** * * @author install1 */ public class SentimentAnalyzerTest implements Callable { private SimilarityMatrix smxParam; private String str; private String str1; private ShiftReduceParser model; private MaxentTagger tagger; private GrammaticalStructureFactory gsf; private StanfordCoreNLP pipeline; private StanfordCoreNLP pipelineSentiment; private StanfordCoreNLP pipelineJMWE; private AbstractSequenceClassifier classifier; public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam) { this.str = str; this.str1 = str1; this.smxParam = smxParam; model = MYSQLDatahandler.getModel(); tagger = MYSQLDatahandler.getTagger(); pipeline = MYSQLDatahandler.getPipeline(); pipelineSentiment = MYSQLDatahandler.getPipelineSentiment(); pipelineJMWE = MYSQLDatahandler.getPipelineJMWE(); gsf = MYSQLDatahandler.getGsf(); classifier = MYSQLDatahandler.getClassifier(); } @Override public SimilarityMatrix call() { try { Double score = -100.0; List> taggedwordlist1 = new ArrayList(); List> taggedwordlist2 = new ArrayList(); DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1)); for (List sentence : tokenizer) { taggedwordlist1.add(model.apply(tagger.tagSentence(sentence)).taggedYield()); } tokenizer = new DocumentPreprocessor(new StringReader(str)); for (List sentence : tokenizer) { taggedwordlist2.add(model.apply(tagger.tagSentence(sentence)).taggedYield()); } int counter = 0; int counter1 = 0; counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum); counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum); int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter; overValue *= 16; score -= overValue; ConcurrentMap tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap(); taggedwordlist1.forEach((TGWList) -> { TGWList.forEach((TaggedWord) -> { if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) { tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag()); } }); }); taggedwordlist1.clear(); AtomicInteger runCount = new AtomicInteger(0); taggedwordlist2.forEach((TGWList) -> { TGWList.forEach((TaggedWord) -> { if (tgwlistIndex.values().contains(TaggedWord.tag())) { tgwlistIndex.values().remove(TaggedWord.tag()); runCount.getAndIncrement(); } }); }); tgwlistIndex.clear(); taggedwordlist2.clear(); score += runCount.get() * 64; Annotation annotation = new Annotation(str1); pipeline.annotate(annotation); ConcurrentMap sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap(); for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) { Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse); } Annotation annotation1 = new Annotation(str); pipeline.annotate(annotation1); for (CoreMap sentence : annotation1.get(CoreAnnotations.SentencesAnnotation.class)) { Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse); Collection allTypedDependencies = gs.allTypedDependencies(); ConcurrentMap filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap(); for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) { Set inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1); Set inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse); ConcurrentMap constiLabels = new MapMaker().concurrencyLevel(2).makeMap(); for (Constituent consti : inT1notT2) { for (Constituent consti1 : inT2notT1) { if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) { score += 64; constiLabels.put(constiLabels.size(), consti.value()); } } } GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1); Collection allTypedDependencies1 = gs1.allTypedDependencies(); for (TypedDependency TDY1 : allTypedDependencies1) { IndexedWord dep = TDY1.dep(); IndexedWord gov = TDY1.gov(); GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep); if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) { score += 900; } GrammaticalRelation reln = TDY1.reln(); if (reln.isApplicable(sentenceConstituencyParse)) { score += 256; } } for (TypedDependency TDY : allTypedDependencies) { IndexedWord dep = TDY.dep(); IndexedWord gov = TDY.gov(); GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep); if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) { score += 900; } GrammaticalRelation reln = TDY.reln(); if (reln.isApplicable(sentenceConstituencyParse1)) { score += 256; } } AtomicInteger runCount1 = new AtomicInteger(0); sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> { sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma()) && !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> { filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma()); return _item; }).forEachOrdered((_item) -> { runCount1.getAndIncrement(); }); }); score += runCount1.get() * 1500; } } sentenceConstituencyParseList.clear(); Annotation annotationSentiment1 = pipelineSentiment.process(str); ConcurrentMap simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap sentiment1 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap sentiment2 = new MapMaker().concurrencyLevel(2).makeMap(); for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) { Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree)); SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); simpleSMXlist.put(simpleSMXlist.size(), predictions); simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector); } annotationSentiment1 = pipelineSentiment.process(str1); for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) { Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree)); SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> { subtracter *= 25; return subtracter; }).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item); for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) { double dot = nodeVector.dot(simpleSMX); double elementSum = nodeVector.kron(simpleSMX).elementSum(); elementSum = Math.round(elementSum * 100.0) / 100.0; if (dot < 0.1) { score += 256; } if (elementSum < 0.1 && elementSum > 0.0) { score += 1300; } else if (elementSum > 0.1 && elementSum < 1.0) { score -= 1100; } else { score -= 1424; } } } score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500; DocumentReaderAndWriter readerAndWriter = classifier.makePlainTextReaderAndWriter(); List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter); List classifyRaw2 = classifier.classifyRaw(str1, readerAndWriter); score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200; Annotation annotationSentiment = pipelineSentiment.process(str); int mainSentiment1 = 0; int longest1 = 0; int mainSentiment2 = 0; int longest2 = 0; for (CoreMap sentence : annotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) { Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); int sentiment = RNNCoreAnnotations.getPredictedClass(tree); String partText = sentence.toString(); SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); if (partText.length() > longest1) { mainSentiment1 = sentiment; longest1 = partText.length(); } } annotationSentiment = pipelineSentiment.process(str1); for (CoreMap sentence : annotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) { Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); int sentiment = RNNCoreAnnotations.getPredictedClass(tree); SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); String partText = sentence.toString(); if (partText.length() > longest2) { mainSentiment2 = sentiment; longest2 = partText.length(); } } if (longest1 != longest2) { long deffLongest = longest1 > longest2 ? longest1 : longest2; long deffshorter = longest1 < longest2 ? longest1 : longest2; if (deffLongest >= (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) { score += (deffLongest - deffshorter) * 200; } else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) { score += (deffLongest - deffshorter) * 200; } else { score -= (deffLongest - deffshorter) * 50; } } Annotation jmweStrAnnotation = new Annotation(str); pipelineJMWE.annotate(jmweStrAnnotation); List sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class); int tokensCounter1 = 0; int tokensCounter2 = 0; int anotatorcounter1 = 0; int anotatorcounter2 = 0; int inflectedCounterPositive1 = 0; int inflectedCounterPositive2 = 0; int inflectedCounterNegative = 0; int MarkedContinuousCounter1 = 0; int MarkedContinuousCounter2 = 0; int UnmarkedPatternCounter = 0; ConcurrentMap ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap strTokenStems2 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap strTokenForm1 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap strTokenForm2 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap strTokenGetEntry1 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap strTokenGetEntry2 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap strTokenGetiPart1 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap strTokenGetiPart2 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap strTokenEntryPOS1 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap strTokenEntryPOS2 = new MapMaker().concurrencyLevel(2).makeMap(); for (CoreMap sentence : sentences) { for (IMWE token : sentence.get(JMWEAnnotation.class)) { if (token.isInflected()) { inflectedCounterPositive1++; } else { inflectedCounterNegative++; } strTokenForm1.put(strTokenForm1.size() + 1, token.getForm()); strTokenGetEntry1.put(strTokenGetEntry1.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1)); Collection values = token.getPartMap().values(); IMWEDesc entry = token.getEntry(); MarkedContinuousCounter1 += entry.getMarkedContinuous(); UnmarkedPatternCounter += entry.getUnmarkedPattern(); for (IMWEDesc.IPart iPart : values) { strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm()); } for (String strPostPrefix : entry.getPOS().getPrefixes()) { strTokenEntryPOS1.put(strTokenEntryPOS1.size() + 1, strPostPrefix); } for (IToken tokens : token.getTokens()) { ITokenMapTag1.put(ITokenMapTag1.size() + 1, tokens.getTag()); for (String strtoken : tokens.getStems()) { strTokenStems1.put(strTokenStems1.size() + 1, strtoken); } } tokensCounter1++; } anotatorcounter1++; } jmweStrAnnotation = new Annotation(str1); pipelineJMWE.annotate(jmweStrAnnotation); sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class); for (CoreMap sentence : sentences) { for (IMWE token : sentence.get(JMWEAnnotation.class)) { if (token.isInflected()) { inflectedCounterPositive2++; } else { inflectedCounterNegative--; } strTokenForm2.put(strTokenForm2.size() + 1, token.getForm()); strTokenGetEntry2.put(strTokenGetEntry2.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1)); Collection values = token.getPartMap().values(); IMWEDesc entry = token.getEntry(); MarkedContinuousCounter2 += entry.getMarkedContinuous(); UnmarkedPatternCounter += entry.getUnmarkedPattern(); for (IMWEDesc.IPart iPart : values) { strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm()); } for (String strPostPrefix : entry.getPOS().getPrefixes()) { strTokenEntryPOS2.put(strTokenEntryPOS2.size() + 1, strPostPrefix); } for (IToken tokens : token.getTokens()) { ITokenMapTag2.put(ITokenMapTag2.size() + 1, tokens.getTag()); for (String strtoken : tokens.getStems()) { strTokenStems2.put(strTokenStems2.size() + 1, strtoken); } } tokensCounter2++; } anotatorcounter2++; } for (String strTokenPos1 : strTokenEntryPOS1.values()) { for (String strTokenPos2 : strTokenEntryPOS2.values()) { if (strTokenPos1.equals(strTokenPos2)) { score += 500; } } } score += UnmarkedPatternCounter * 1600; if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) { score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500 : (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500; } for (String strTokeniPart1 : strTokenGetiPart1.values()) { for (String strTokeniPart2 : strTokenGetiPart2.values()) { if (strTokeniPart1.equals(strTokeniPart2)) { score += 400; } } } for (String strTokenEntry1 : strTokenGetEntry1.values()) { for (String strTokenEntry2 : strTokenGetEntry2.values()) { if (strTokenEntry1.equals(strTokenEntry2)) { score += 2500; } } } for (String strmapTag : ITokenMapTag1.values()) { for (String strmapTag1 : ITokenMapTag2.values()) { if (strmapTag.equals(strmapTag1)) { score += 1450; } } } for (String strTokenForm1itr1 : strTokenForm1.values()) { for (String strTokenForm1itr2 : strTokenForm2.values()) { if (strTokenForm1itr1.equals(strTokenForm1itr2)) { score += 2600; } else if (strTokenForm1itr1.contains(strTokenForm1itr2)) { score += 500; } } } for (String strTokenStem : strTokenStems1.values()) { for (String strTokenStem1 : strTokenStems2.values()) { if (strTokenStem.equals(strTokenStem1)) { score += 1500; } } } if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) { score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650; } if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) { score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550; } if (anotatorcounter1 > 1 && anotatorcounter2 > 1) { score += (anotatorcounter1 + anotatorcounter2) * 400; } if (tokensCounter1 > 0 && tokensCounter2 > 0) { score += (tokensCounter1 + tokensCounter2) * 400; } else { score -= tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500; } LevenshteinDistance leven = new LevenshteinDistance(str, str1); int SentenceScoreDiff = leven.computeLevenshteinDistance(); SentenceScoreDiff *= 15; score -= SentenceScoreDiff; System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n"); smxParam.setDistance(score); } catch (Exception ex) { System.out.println("ex: " + ex.getMessage() + "\n"); smxParam.setDistance(-1000); return smxParam; } return smxParam; } }