slight moving, predominantly calculation updates
This commit is contained in:
jenzur
parent
feaf9b0adc
commit
d788d76ac6
@ -122,7 +122,7 @@ public class DataMapper {
|
||||
l_cCon = DBCPDataSource.getConnection();
|
||||
l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
|
||||
java.sql.ResultSet.CONCUR_READ_ONLY);
|
||||
l_pStatement.setFetchSize(Integer.MIN_VALUE);
|
||||
l_pStatement.setFetchSize(0);
|
||||
System.out.println("Matrix update size: " + WS4JListUpdate.size());
|
||||
for (SimilarityMatrix ws4j : WS4JListUpdate.values()) {
|
||||
l_pStatement.setString(1, ws4j.getPrimaryString());
|
||||
@ -146,8 +146,9 @@ public class DataMapper {
|
||||
String l_sSQL = "SELECT * FROM `WordMatrix`";
|
||||
try (PreparedStatement l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
|
||||
java.sql.ResultSet.CONCUR_READ_ONLY)) {
|
||||
l_pStatement.setFetchSize(Integer.MIN_VALUE);
|
||||
l_pStatement.setFetchSize(0);
|
||||
try (ResultSet l_rsSearch = l_pStatement.executeQuery()) {
|
||||
l_rsSearch.setFetchSize(0);
|
||||
int i = 0;
|
||||
LinkedHashMap<String, Double> LHMLocal = new LinkedHashMap();
|
||||
while (l_rsSearch.next()) {
|
||||
|
||||
@ -217,7 +217,7 @@ public class Datahandler {
|
||||
public void addHLstatsMessages() {
|
||||
ConcurrentMap<Integer, String> hlStatsMessages = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strCacheLocal = stringCache;
|
||||
int hardcap = 8500;
|
||||
int hardcap = 55000;
|
||||
int ij = 0;
|
||||
for (String str : DataMapper.getHLstatsMessages().values()) {
|
||||
hlStatsMessages.put(ij, str);
|
||||
@ -483,7 +483,7 @@ public class Datahandler {
|
||||
|
||||
public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings) {
|
||||
similar = "";
|
||||
minDistance = 7.5;
|
||||
minDistance = 12.5;
|
||||
concurrentStrings.values().parallelStream().forEach((str) -> {
|
||||
LevenshteinDistance leven = new LevenshteinDistance(toBeCompared, str);
|
||||
double distance = leven.computeLevenshteinDistance();
|
||||
@ -630,8 +630,8 @@ public class Datahandler {
|
||||
}
|
||||
|
||||
private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
|
||||
ShiftReduceParser model = getModel();
|
||||
MaxentTagger tagger = getTagger();
|
||||
ShiftReduceParser modelLocal = getModel();
|
||||
MaxentTagger taggerLocal = getTagger();
|
||||
ConcurrentMap<Integer, String> strreturn = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
str.values().parallelStream().forEach(str1 -> {
|
||||
ConcurrentMap<Integer, String> TGWList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
@ -646,8 +646,8 @@ public class Datahandler {
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
int counter = 0;
|
||||
List<TaggedWord> taggedWords;
|
||||
List<TaggedWord> tagged1 = tagger.tagSentence(sentence);
|
||||
Tree tree = model.apply(tagged1);
|
||||
List<TaggedWord> tagged1 = taggerLocal.tagSentence(sentence);
|
||||
Tree tree = modelLocal.apply(tagged1);
|
||||
taggedWords = tree.taggedYield();
|
||||
for (TaggedWord TGW : taggedWords) {
|
||||
if (!TGWList.values().contains(TGW.tag()) && !TGW.tag().equals(":") && !TGW.word().equals(TGW.tag())) {
|
||||
@ -659,7 +659,6 @@ public class Datahandler {
|
||||
ConcurrentMap<Integer, Word> wordList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (Word lab : tree.yieldWords()) {
|
||||
if (lab != null && lab.word() != null) {
|
||||
//System.out.println("lab: " + lab + " \n");
|
||||
if (!wordList.values().contains(lab) && lab.value() != null && !lab.value().equals(":")) {
|
||||
wordList.put(wordList.size() + 1, lab);
|
||||
addCounter++;
|
||||
@ -681,7 +680,7 @@ public class Datahandler {
|
||||
for (String strVals : values) {
|
||||
LevenshteinDistance leven = new LevenshteinDistance(strVals, str1);
|
||||
double Distance = leven.computeLevenshteinDistance();
|
||||
int maxpermittedDistance = 2;
|
||||
int maxpermittedDistance = 5;
|
||||
if (Distance < maxpermittedDistance) {
|
||||
tooclosematch = true;
|
||||
break;
|
||||
|
||||
@ -5,12 +5,6 @@
|
||||
*/
|
||||
package FunctionLayer;
|
||||
|
||||
import com.google.common.collect.MapMaker;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
|
||||
@ -38,11 +38,14 @@ import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.OptionalDouble;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.BinaryOperator;
|
||||
import java.util.function.Function;
|
||||
import org.ejml.simple.SimpleMatrix;
|
||||
|
||||
/*
|
||||
@ -96,378 +99,517 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||
Double score = -100.0;
|
||||
try {
|
||||
List<List<TaggedWord>> taggedwordlist1 = new ArrayList();
|
||||
List<List<TaggedWord>> taggedwordlist2 = new ArrayList();
|
||||
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
|
||||
//noneDelete
|
||||
TokenizerFactory<CoreLabel> ptbTokenizerFactory
|
||||
= PTBTokenizer.factory(new CoreLabelTokenFactory(), "untokenizable=firstDelete");
|
||||
tokenizer.setTokenizerFactory(ptbTokenizerFactory);
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
taggedwordlist1.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
||||
}
|
||||
tokenizer = new DocumentPreprocessor(new StringReader(str));
|
||||
tokenizer.setTokenizerFactory(ptbTokenizerFactory);
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
taggedwordlist2.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
||||
}
|
||||
int counter = 0;
|
||||
int counter1 = 0;
|
||||
counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum);
|
||||
counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum);
|
||||
int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter;
|
||||
overValue *= 32;
|
||||
score -= overValue;
|
||||
ConcurrentMap<Integer, String> tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
taggedwordlist1.forEach((TGWList) -> {
|
||||
TGWList.forEach((TaggedWord) -> {
|
||||
if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) {
|
||||
tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag());
|
||||
}
|
||||
});
|
||||
});
|
||||
AtomicInteger runCount = new AtomicInteger(0);
|
||||
taggedwordlist2.forEach((TGWList) -> {
|
||||
TGWList.forEach((TaggedWord) -> {
|
||||
if (tgwlistIndex.values().contains(TaggedWord.tag())) {
|
||||
tgwlistIndex.values().remove(TaggedWord.tag());
|
||||
runCount.getAndIncrement();
|
||||
}
|
||||
});
|
||||
});
|
||||
score += runCount.get() * 64;
|
||||
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
try {
|
||||
for (CoreMap sentence : pipelineAnnotation1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||
sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse);
|
||||
List<List<TaggedWord>> taggedwordlist2 = new ArrayList();
|
||||
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
|
||||
//noneDelete
|
||||
TokenizerFactory<CoreLabel> ptbTokenizerFactory
|
||||
= PTBTokenizer.factory(new CoreLabelTokenFactory(), "untokenizable=firstDelete");
|
||||
tokenizer.setTokenizerFactory(ptbTokenizerFactory);
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
taggedwordlist1.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
||||
}
|
||||
for (CoreMap sentence : pipelineAnnotation2.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
int constiRelationsize = 0;
|
||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||
GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse);
|
||||
Collection<TypedDependency> allTypedDependencies = gs.allTypedDependencies();
|
||||
ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) {
|
||||
Set<Constituent> constinuent1 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1);
|
||||
Set<Constituent> constinuent2 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse);
|
||||
ConcurrentMap<Integer, String> constiLabels = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (Constituent consti : constinuent1) {
|
||||
for (Constituent consti1 : constinuent2) {
|
||||
if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) {
|
||||
constiLabels.put(constiLabels.size(), consti.value());
|
||||
constiRelationsize++;
|
||||
tokenizer = new DocumentPreprocessor(new StringReader(str));
|
||||
tokenizer.setTokenizerFactory(ptbTokenizerFactory);
|
||||
for (List<HasWord> sentence : tokenizer) {
|
||||
taggedwordlist2.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
||||
}
|
||||
int counter = 0;
|
||||
int counter1 = 0;
|
||||
counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum);
|
||||
counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum);
|
||||
int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter;
|
||||
overValue *= 32;
|
||||
score -= overValue;
|
||||
ConcurrentMap<Integer, String> tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
taggedwordlist1.forEach((TGWList) -> {
|
||||
TGWList.forEach((TaggedWord) -> {
|
||||
if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) {
|
||||
tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag());
|
||||
}
|
||||
});
|
||||
});
|
||||
AtomicInteger runCount = new AtomicInteger(0);
|
||||
taggedwordlist2.forEach((TGWList) -> {
|
||||
TGWList.forEach((TaggedWord) -> {
|
||||
if (tgwlistIndex.values().contains(TaggedWord.tag())) {
|
||||
tgwlistIndex.values().remove(TaggedWord.tag());
|
||||
runCount.getAndIncrement();
|
||||
}
|
||||
});
|
||||
});
|
||||
score += runCount.get() * 64;
|
||||
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
try {
|
||||
for (CoreMap sentence : pipelineAnnotation1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||
sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse);
|
||||
}
|
||||
for (CoreMap sentence : pipelineAnnotation2.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
int constiRelationsize = 0;
|
||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||
GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse);
|
||||
Collection<TypedDependency> allTypedDependencies = gs.allTypedDependencies();
|
||||
ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) {
|
||||
Set<Constituent> constinuent1 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1);
|
||||
Set<Constituent> constinuent2 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse);
|
||||
ConcurrentMap<Integer, String> constiLabels = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (Constituent consti : constinuent1) {
|
||||
for (Constituent consti1 : constinuent2) {
|
||||
if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) {
|
||||
constiLabels.put(constiLabels.size(), consti.value());
|
||||
constiRelationsize++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
int constituents1 = constinuent1.size() - constiRelationsize;
|
||||
int constituents2 = constinuent2.size() - constiRelationsize;
|
||||
if (constituents1 > 0 && constituents2 > 0) {
|
||||
score -= (constituents1 + constituents2) * 200;
|
||||
} else {
|
||||
score += constiRelationsize * 200;
|
||||
}
|
||||
GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1);
|
||||
Collection<TypedDependency> allTypedDependencies1 = gs1.allTypedDependencies();
|
||||
for (TypedDependency TDY1 : allTypedDependencies1) {
|
||||
IndexedWord dep = TDY1.dep();
|
||||
IndexedWord gov = TDY1.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 1900;
|
||||
int constituents1 = constinuent1.size() - constiRelationsize;
|
||||
int constituents2 = constinuent2.size() - constiRelationsize;
|
||||
if (constituents1 > 0 && constituents2 > 0) {
|
||||
score -= (constituents1 + constituents2) * 200;
|
||||
} else {
|
||||
score += constiRelationsize * 200;
|
||||
}
|
||||
GrammaticalRelation reln = TDY1.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 525;
|
||||
GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1);
|
||||
Collection<TypedDependency> allTypedDependencies1 = gs1.allTypedDependencies();
|
||||
for (TypedDependency TDY1 : allTypedDependencies1) {
|
||||
IndexedWord dep = TDY1.dep();
|
||||
IndexedWord gov = TDY1.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 1900;
|
||||
}
|
||||
GrammaticalRelation reln = TDY1.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 525;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (TypedDependency TDY : allTypedDependencies) {
|
||||
IndexedWord dep = TDY.dep();
|
||||
IndexedWord gov = TDY.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 900;
|
||||
for (TypedDependency TDY : allTypedDependencies) {
|
||||
IndexedWord dep = TDY.dep();
|
||||
IndexedWord gov = TDY.gov();
|
||||
GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep);
|
||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||
score += 900;
|
||||
}
|
||||
GrammaticalRelation reln = TDY.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse1)) {
|
||||
score += 525;
|
||||
}
|
||||
}
|
||||
GrammaticalRelation reln = TDY.reln();
|
||||
if (reln.isApplicable(sentenceConstituencyParse1)) {
|
||||
score += 525;
|
||||
}
|
||||
}
|
||||
AtomicInteger runCount1 = new AtomicInteger(0);
|
||||
sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> {
|
||||
sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma())
|
||||
&& !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> {
|
||||
filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma());
|
||||
return _item;
|
||||
}).forEachOrdered((_item) -> {
|
||||
runCount1.getAndIncrement();
|
||||
AtomicInteger runCount1 = new AtomicInteger(0);
|
||||
sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> {
|
||||
sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma())
|
||||
&& !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> {
|
||||
filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma());
|
||||
return _item;
|
||||
}).forEachOrdered((_item) -> {
|
||||
runCount1.getAndIncrement();
|
||||
});
|
||||
});
|
||||
});
|
||||
score += runCount1.get() * 1500;
|
||||
score += runCount1.get() * 1500;
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
System.out.println("pipelineAnnotation stacktrace: " + ex.getLocalizedMessage() + "\n");
|
||||
}
|
||||
sentenceConstituencyParseList.clear();
|
||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Integer> sentiment1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Integer> sentiment2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree));
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||
simpleSMXlist.put(simpleSMXlist.size(), predictions);
|
||||
simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector);
|
||||
}
|
||||
ConcurrentMap<Integer, Double> elementSumCounter = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Double> dotMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree));
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||
ConcurrentMap<Integer, Double> AccumulateDotMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Double> subtractorMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
score += simpleSMXlist.values().stream().map(new Function<SimpleMatrix, Double>() {
|
||||
@Override
|
||||
public Double apply(SimpleMatrix simpleSMX) {
|
||||
return predictions.dot(simpleSMX) * 100;
|
||||
}
|
||||
}).map(new Function<Double, Double>() {
|
||||
@Override
|
||||
public Double apply(Double dot) {
|
||||
AccumulateDotMap.put(AccumulateDotMap.size() + 1, dot);
|
||||
return dot > 50 ? dot - 100 : dot > 0 ? 100 - dot : 0;
|
||||
}
|
||||
}).map((subtracter) -> {
|
||||
subtractorMap.put(subtractorMap.size() + 1, subtracter);
|
||||
subtracter *= 25; //25
|
||||
return subtracter;
|
||||
}).map(new Function<Double, Double>() {
|
||||
@Override
|
||||
public Double apply(Double subtracter) {
|
||||
return subtracter;
|
||||
}
|
||||
}).reduce(score, new BinaryOperator<Double>() {
|
||||
@Override
|
||||
public Double apply(Double accumulator, Double _item) {
|
||||
int accumulator1 = 0;
|
||||
while (accumulator < 0) {
|
||||
accumulator++;
|
||||
accumulator1++;
|
||||
}
|
||||
return accumulator1 + _item;
|
||||
}
|
||||
});
|
||||
Double subTracPre = 0.0;
|
||||
for (Double subtractors : subtractorMap.values()) {
|
||||
if (Objects.equals(subTracPre, subtractors)) {
|
||||
score -= 2000;
|
||||
}
|
||||
subTracPre = subtractors;
|
||||
}
|
||||
score += simpleSMXlist.values().stream().map(new Function<SimpleMatrix, Double>() {
|
||||
@Override
|
||||
public Double apply(SimpleMatrix simpleSMX) {
|
||||
return simpleSMX.dot(predictions) * 100;
|
||||
}
|
||||
}).map(new Function<Double, Double>() {
|
||||
@Override
|
||||
public Double apply(Double dot) {
|
||||
AccumulateDotMap.put(AccumulateDotMap.size() + 1, dot);
|
||||
return dot > 50 ? dot - 50 : dot > 0 ? 50 - dot : 0;
|
||||
}
|
||||
}).map((subtracter) -> {
|
||||
subtracter *= 25; //25
|
||||
return subtracter;
|
||||
}).map(new Function<Double, Double>() {
|
||||
@Override
|
||||
public Double apply(Double subtracter) {
|
||||
return subtracter;
|
||||
}
|
||||
}).reduce(score, new BinaryOperator<Double>() {
|
||||
@Override
|
||||
public Double apply(Double accumulator, Double _item) {
|
||||
int accumulator1 = 0;
|
||||
while (accumulator < 0) {
|
||||
accumulator++;
|
||||
accumulator1++;
|
||||
}
|
||||
return accumulator1 + _item;
|
||||
}
|
||||
});
|
||||
Double preAccumulatorDot = 0.0;
|
||||
Double postAccumulatorDot = 0.0;
|
||||
for (Double accumulators : AccumulateDotMap.values()) {
|
||||
if (preAccumulatorDot == accumulators) {
|
||||
if (postAccumulatorDot == accumulators) {
|
||||
score -= 4000;
|
||||
}
|
||||
postAccumulatorDot = accumulators;
|
||||
}
|
||||
preAccumulatorDot = accumulators;
|
||||
}
|
||||
subTracPre = 0.0;
|
||||
for (Double subtractors : subtractorMap.values()) {
|
||||
if (Objects.equals(subTracPre, subtractors)) {
|
||||
score -= 2000;
|
||||
}
|
||||
subTracPre = subtractors;
|
||||
}
|
||||
Double preDot = 0.0;
|
||||
Double postDot = 0.0;
|
||||
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
|
||||
double dot = nodeVector.dot(simpleSMX);
|
||||
double elementSum = nodeVector.kron(simpleSMX).elementSum();
|
||||
if (preDot == dot) {
|
||||
if (postDot == dot) {
|
||||
score -= 4000;
|
||||
}
|
||||
postDot = dot;
|
||||
}
|
||||
preDot = dot;
|
||||
elementSum = Math.round(elementSum * 100.0) / 100.0;
|
||||
elementSumCounter.put(elementSumCounter.size() + 1, elementSum);
|
||||
dotMap.put(dotMap.size() + 1, dot);
|
||||
if (dot < 0.1) {
|
||||
score += 256;
|
||||
}
|
||||
if (dot > 0.50) {
|
||||
score -= 2400;
|
||||
}
|
||||
if (elementSum < 0.01 && elementSum > 0.00) {
|
||||
score += 3300;
|
||||
} else if (elementSum > 0.1 && elementSum < 0.2) {
|
||||
score += 1100;
|
||||
} else {
|
||||
score -= elementSum * 1424;
|
||||
}
|
||||
}
|
||||
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
|
||||
double dot = simpleSMX.dot(nodeVector);
|
||||
double elementSum = simpleSMX.kron(nodeVector).elementSum();
|
||||
if (preDot == dot) {
|
||||
if (postDot == dot) {
|
||||
score -= 4000;
|
||||
}
|
||||
postDot = dot;
|
||||
}
|
||||
preDot = dot;
|
||||
elementSum = Math.round(elementSum * 100.0) / 100.0;
|
||||
elementSumCounter.put(elementSumCounter.size() + 1, elementSum);
|
||||
dotMap.put(dotMap.size() + 1, dot);
|
||||
if (dot < 0.1) {
|
||||
score += 256;
|
||||
}
|
||||
if (dot > 0.50) {
|
||||
score -= 2400;
|
||||
}
|
||||
if (elementSum < 0.01 && elementSum > 0.00) {
|
||||
score += 1300;
|
||||
} else if (elementSum > 0.1 && elementSum < 1.0) {
|
||||
score += 1100;
|
||||
} else {
|
||||
score -= elementSum * 1424;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
System.out.println("pipelineAnnotation stacktrace: " + ex.getLocalizedMessage() + "\n");
|
||||
}
|
||||
sentenceConstituencyParseList.clear();
|
||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Integer> sentiment1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Integer> sentiment2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree));
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||
simpleSMXlist.put(simpleSMXlist.size(), predictions);
|
||||
simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector);
|
||||
}
|
||||
ConcurrentMap<Integer, Double> elementSumCounter = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, Double> dotMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree));
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||
score += simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : dot > 0 ? 50 - dot : 50).map((subtracter) -> {
|
||||
subtracter *= 25; //25
|
||||
return subtracter;
|
||||
}).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator + _item);
|
||||
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
|
||||
double dot = nodeVector.dot(simpleSMX);
|
||||
double elementSum = nodeVector.kron(simpleSMX).elementSum();
|
||||
elementSum = Math.round(elementSum * 100.0) / 100.0;
|
||||
elementSumCounter.put(elementSumCounter.size() + 1, elementSum);
|
||||
dotMap.put(dotMap.size() + 1, dot);
|
||||
if (dot < 0.1) {
|
||||
score += 256;
|
||||
}
|
||||
if (dot > 0.50) {
|
||||
score -= 2400;
|
||||
}
|
||||
if (elementSum < 0.01 && elementSum > 0.00) {
|
||||
score += 1300;
|
||||
} else if (elementSum > 0.1 && elementSum < 1.0) {
|
||||
score += 1100;
|
||||
} else {
|
||||
score -= elementSum * 1424;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (dotMap.values().size() > 1) {
|
||||
OptionalDouble minvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).min();
|
||||
OptionalDouble maxvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).max();
|
||||
if (maxvalueDots.getAsDouble() - minvalueDots.getAsDouble() < 0.05) {
|
||||
score += 3500;
|
||||
double total = minvalueDots.getAsDouble() + maxvalueDots.getAsDouble();
|
||||
boolean permitted = false;
|
||||
if (minvalueDots.getAsDouble() != maxvalueDots.getAsDouble()) {
|
||||
permitted = true;
|
||||
}
|
||||
if (permitted) {
|
||||
Double dotsVariance = maxvalueDots.getAsDouble() - minvalueDots.getAsDouble();
|
||||
if (maxvalueDots.getAsDouble() > minvalueDots.getAsDouble() * 10) {
|
||||
score -= 5500;
|
||||
} else if (minvalueDots.getAsDouble() < -0.10) {
|
||||
score -= 3500;
|
||||
} else if (dotsVariance < 0.5) {
|
||||
score += 3500;
|
||||
} else if (dotsVariance > minvalueDots.getAsDouble() * 2) {
|
||||
score += 3500;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (elementSumCounter.values().size() > 1){
|
||||
OptionalDouble minvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).min();
|
||||
OptionalDouble maxvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).max();
|
||||
if (maxvalueElements.getAsDouble() - minvalueElements.getAsDouble() < 0.05) {
|
||||
Double elementsVariance = maxvalueElements.getAsDouble() - minvalueElements.getAsDouble();
|
||||
if (elementsVariance < 0.05 && maxvalueElements.getAsDouble() > 0.0 && minvalueElements.getAsDouble() > 0.0 && elementsVariance > 0.000) {
|
||||
score += 3500;
|
||||
} else if (minvalueElements.getAsDouble() < 0.0 && minvalueElements.getAsDouble() - maxvalueElements.getAsDouble() < 0.50) {
|
||||
score -= 2500;
|
||||
}
|
||||
}
|
||||
score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500;
|
||||
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
|
||||
List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter);
|
||||
List classifyRaw2 = classifier.classifyRaw(str1, readerAndWriter);
|
||||
score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200;
|
||||
int mainSentiment1 = 0;
|
||||
int longest1 = 0;
|
||||
int mainSentiment2 = 0;
|
||||
int longest2 = 0;
|
||||
for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
||||
String partText = sentence.toString();
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
if (partText.length() > longest1) {
|
||||
mainSentiment1 = sentiment;
|
||||
longest1 = partText.length();
|
||||
}
|
||||
}
|
||||
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
String partText = sentence.toString();
|
||||
if (partText.length() > longest2) {
|
||||
mainSentiment2 = sentiment;
|
||||
longest2 = partText.length();
|
||||
}
|
||||
}
|
||||
if (longest1 != longest2) {
|
||||
long deffLongest = longest1 > longest2 ? longest1 : longest2;
|
||||
long deffshorter = longest1 < longest2 ? longest1 : longest2;
|
||||
//deffLongest >= (deffshorter * 2)
|
||||
if (deffLongest < (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
|
||||
score += (deffLongest - deffshorter) * 120;
|
||||
} else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) {
|
||||
score += (deffLongest - deffshorter) * 120;
|
||||
} else if (deffLongest - deffshorter < 2) {
|
||||
score += (deffLongest + deffshorter) * 40;
|
||||
} else if (deffLongest - deffshorter <= 5){
|
||||
score += 2500;
|
||||
} else{
|
||||
score -= (deffLongest - deffshorter) * 50;
|
||||
}
|
||||
}
|
||||
int tokensCounter1 = 0;
|
||||
int tokensCounter2 = 0;
|
||||
int anotatorcounter1 = 0;
|
||||
int anotatorcounter2 = 0;
|
||||
int inflectedCounterPositive1 = 0;
|
||||
int inflectedCounterPositive2 = 0;
|
||||
int inflectedCounterNegative = 0;
|
||||
int MarkedContinuousCounter1 = 0;
|
||||
int MarkedContinuousCounter2 = 0;
|
||||
int UnmarkedPatternCounter = 0;
|
||||
ConcurrentMap<Integer, String> ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenStems2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenForm1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenForm2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetEntry1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetEntry2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetiPart1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetiPart2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenEntryPOS1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenEntryPOS2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
try {
|
||||
List<CoreMap> sentences = jmweStrAnnotation1.get(CoreAnnotations.SentencesAnnotation.class);
|
||||
for (CoreMap sentence : sentences) {
|
||||
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
||||
if (token.isInflected()) {
|
||||
inflectedCounterPositive1++;
|
||||
} else {
|
||||
inflectedCounterNegative++;
|
||||
}
|
||||
strTokenForm1.put(strTokenForm1.size() + 1, token.getForm());
|
||||
strTokenGetEntry1.put(strTokenGetEntry1.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||
IMWEDesc entry = token.getEntry();
|
||||
MarkedContinuousCounter1 += entry.getMarkedContinuous();
|
||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||
for (IMWEDesc.IPart iPart : values) {
|
||||
strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm());
|
||||
}
|
||||
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
||||
strTokenEntryPOS1.put(strTokenEntryPOS1.size() + 1, strPostPrefix);
|
||||
}
|
||||
for (IToken tokens : token.getTokens()) {
|
||||
ITokenMapTag1.put(ITokenMapTag1.size() + 1, tokens.getTag());
|
||||
for (String strtoken : tokens.getStems()) {
|
||||
strTokenStems1.put(strTokenStems1.size() + 1, strtoken);
|
||||
}
|
||||
}
|
||||
tokensCounter1++;
|
||||
}
|
||||
anotatorcounter1++;
|
||||
}
|
||||
sentences = jmweStrAnnotation2.get(CoreAnnotations.SentencesAnnotation.class);
|
||||
for (CoreMap sentence : sentences) {
|
||||
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
||||
if (token.isInflected()) {
|
||||
inflectedCounterPositive2++;
|
||||
} else {
|
||||
inflectedCounterNegative--;
|
||||
}
|
||||
strTokenForm2.put(strTokenForm2.size() + 1, token.getForm());
|
||||
strTokenGetEntry2.put(strTokenGetEntry2.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||
IMWEDesc entry = token.getEntry();
|
||||
MarkedContinuousCounter2 += entry.getMarkedContinuous();
|
||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||
for (IMWEDesc.IPart iPart : values) {
|
||||
strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm());
|
||||
}
|
||||
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
||||
strTokenEntryPOS2.put(strTokenEntryPOS2.size() + 1, strPostPrefix);
|
||||
}
|
||||
for (IToken tokens : token.getTokens()) {
|
||||
ITokenMapTag2.put(ITokenMapTag2.size() + 1, tokens.getTag());
|
||||
for (String strtoken : tokens.getStems()) {
|
||||
strTokenStems2.put(strTokenStems2.size() + 1, strtoken);
|
||||
}
|
||||
}
|
||||
tokensCounter2++;
|
||||
}
|
||||
anotatorcounter2++;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
System.out.println("SENTIMENT stacktrace: " + ex.getMessage() + "\n");
|
||||
}
|
||||
for (String strTokenPos1 : strTokenEntryPOS1.values()) {
|
||||
for (String strTokenPos2 : strTokenEntryPOS2.values()) {
|
||||
if (strTokenPos1.equals(strTokenPos2)) {
|
||||
score += 500;
|
||||
score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500;
|
||||
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
|
||||
List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter);
|
||||
List classifyRaw2 = classifier.classifyRaw(str1, readerAndWriter);
|
||||
score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200;
|
||||
int mainSentiment1 = 0;
|
||||
int longest1 = 0;
|
||||
int mainSentiment2 = 0;
|
||||
int longest2 = 0;
|
||||
for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
||||
String partText = sentence.toString();
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
if (partText.length() > longest1) {
|
||||
mainSentiment1 = sentiment;
|
||||
longest1 = partText.length();
|
||||
}
|
||||
}
|
||||
}
|
||||
if (UnmarkedPatternCounter > 0 && UnmarkedPatternCounter < 5) {
|
||||
score += UnmarkedPatternCounter * 1600;
|
||||
}
|
||||
if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) {
|
||||
score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500
|
||||
: (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500;
|
||||
}
|
||||
for (String strTokeniPart1 : strTokenGetiPart1.values()) {
|
||||
for (String strTokeniPart2 : strTokenGetiPart2.values()) {
|
||||
if (strTokeniPart1.equals(strTokeniPart2)) {
|
||||
score += 400;
|
||||
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||
String partText = sentence.toString();
|
||||
if (partText.length() > longest2) {
|
||||
mainSentiment2 = sentiment;
|
||||
longest2 = partText.length();
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenEntry1 : strTokenGetEntry1.values()) {
|
||||
for (String strTokenEntry2 : strTokenGetEntry2.values()) {
|
||||
if (strTokenEntry1.equals(strTokenEntry2)) {
|
||||
if (longest1 != longest2) {
|
||||
long deffLongest = longest1 > longest2 ? longest1 : longest2;
|
||||
long deffshorter = longest1 < longest2 ? longest1 : longest2;
|
||||
if (deffLongest > deffshorter * 5) {
|
||||
score -= 5500;
|
||||
} else if (deffLongest < (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
|
||||
score += (deffLongest - deffshorter) * 120;
|
||||
} else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) {
|
||||
score += (deffLongest - deffshorter) * 120;
|
||||
} else if (deffLongest - deffshorter < 2) {
|
||||
score += (deffLongest + deffshorter) * 40;
|
||||
} else {
|
||||
score -= (deffLongest - deffshorter) * 50;
|
||||
}
|
||||
if (deffLongest - deffshorter <= 5) {
|
||||
score += 2500;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strmapTag : ITokenMapTag1.values()) {
|
||||
for (String strmapTag1 : ITokenMapTag2.values()) {
|
||||
if (strmapTag.equals(strmapTag1)) {
|
||||
score += 1450;
|
||||
int tokensCounter1 = 0;
|
||||
int tokensCounter2 = 0;
|
||||
int anotatorcounter1 = 0;
|
||||
int anotatorcounter2 = 0;
|
||||
int inflectedCounterPositive1 = 0;
|
||||
int inflectedCounterPositive2 = 0;
|
||||
int inflectedCounterNegative = 0;
|
||||
int MarkedContinuousCounter1 = 0;
|
||||
int MarkedContinuousCounter2 = 0;
|
||||
int UnmarkedPatternCounter = 0;
|
||||
ConcurrentMap<Integer, String> ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenStems2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenForm1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenForm2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetEntry1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetEntry2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetiPart1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenGetiPart2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenEntryPOS1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strTokenEntryPOS2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
try {
|
||||
List<CoreMap> sentences = jmweStrAnnotation1.get(CoreAnnotations.SentencesAnnotation.class);
|
||||
for (CoreMap sentence : sentences) {
|
||||
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
||||
if (token.isInflected()) {
|
||||
inflectedCounterPositive1++;
|
||||
} else {
|
||||
inflectedCounterNegative++;
|
||||
}
|
||||
strTokenForm1.put(strTokenForm1.size() + 1, token.getForm());
|
||||
strTokenGetEntry1.put(strTokenGetEntry1.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||
IMWEDesc entry = token.getEntry();
|
||||
MarkedContinuousCounter1 += entry.getMarkedContinuous();
|
||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||
for (IMWEDesc.IPart iPart : values) {
|
||||
strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm());
|
||||
}
|
||||
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
||||
strTokenEntryPOS1.put(strTokenEntryPOS1.size() + 1, strPostPrefix);
|
||||
}
|
||||
for (IToken tokens : token.getTokens()) {
|
||||
ITokenMapTag1.put(ITokenMapTag1.size() + 1, tokens.getTag());
|
||||
for (String strtoken : tokens.getStems()) {
|
||||
strTokenStems1.put(strTokenStems1.size() + 1, strtoken);
|
||||
}
|
||||
}
|
||||
tokensCounter1++;
|
||||
}
|
||||
anotatorcounter1++;
|
||||
}
|
||||
sentences = jmweStrAnnotation2.get(CoreAnnotations.SentencesAnnotation.class);
|
||||
for (CoreMap sentence : sentences) {
|
||||
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
||||
if (token.isInflected()) {
|
||||
inflectedCounterPositive2++;
|
||||
} else {
|
||||
inflectedCounterNegative--;
|
||||
}
|
||||
strTokenForm2.put(strTokenForm2.size() + 1, token.getForm());
|
||||
strTokenGetEntry2.put(strTokenGetEntry2.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||
IMWEDesc entry = token.getEntry();
|
||||
MarkedContinuousCounter2 += entry.getMarkedContinuous();
|
||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||
for (IMWEDesc.IPart iPart : values) {
|
||||
strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm());
|
||||
}
|
||||
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
||||
strTokenEntryPOS2.put(strTokenEntryPOS2.size() + 1, strPostPrefix);
|
||||
}
|
||||
for (IToken tokens : token.getTokens()) {
|
||||
ITokenMapTag2.put(ITokenMapTag2.size() + 1, tokens.getTag());
|
||||
for (String strtoken : tokens.getStems()) {
|
||||
strTokenStems2.put(strTokenStems2.size() + 1, strtoken);
|
||||
}
|
||||
}
|
||||
tokensCounter2++;
|
||||
}
|
||||
anotatorcounter2++;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
System.out.println("SENTIMENT stacktrace: " + ex.getMessage() + "\n");
|
||||
}
|
||||
for (String strTokenPos1 : strTokenEntryPOS1.values()) {
|
||||
for (String strTokenPos2 : strTokenEntryPOS2.values()) {
|
||||
if (strTokenPos1.equals(strTokenPos2)) {
|
||||
score += 500;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenForm1itr1 : strTokenForm1.values()) {
|
||||
for (String strTokenForm1itr2 : strTokenForm2.values()) {
|
||||
if (strTokenForm1itr1.equals(strTokenForm1itr2)) {
|
||||
score += 2600;
|
||||
} else if (strTokenForm1itr1.contains(strTokenForm1itr2)) {
|
||||
score += 500;
|
||||
if (UnmarkedPatternCounter > 0 && UnmarkedPatternCounter < 5) {
|
||||
score += UnmarkedPatternCounter * 1600;
|
||||
}
|
||||
if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) {
|
||||
score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500
|
||||
: (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500;
|
||||
}
|
||||
for (String strTokeniPart1 : strTokenGetiPart1.values()) {
|
||||
for (String strTokeniPart2 : strTokenGetiPart2.values()) {
|
||||
if (strTokeniPart1.equals(strTokeniPart2)) {
|
||||
score += 400;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenStem : strTokenStems1.values()) {
|
||||
for (String strTokenStem1 : strTokenStems2.values()) {
|
||||
if (strTokenStem.equals(strTokenStem1)) {
|
||||
score += 1500;
|
||||
for (String strTokenEntry1 : strTokenGetEntry1.values()) {
|
||||
for (String strTokenEntry2 : strTokenGetEntry2.values()) {
|
||||
if (strTokenEntry1.equals(strTokenEntry2)) {
|
||||
score += 2500;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) {
|
||||
score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650;
|
||||
}
|
||||
if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) {
|
||||
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
|
||||
}
|
||||
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
|
||||
score += (anotatorcounter1 + anotatorcounter2) * 400;
|
||||
}
|
||||
if (tokensCounter1 > 0 && tokensCounter2 > 0) {
|
||||
score += (tokensCounter1 + tokensCounter2) * 400;
|
||||
} else {
|
||||
int elseint = tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
||||
score -= elseint;
|
||||
}
|
||||
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
|
||||
double SentenceScoreDiff = leven.computeLevenshteinDistance();
|
||||
SentenceScoreDiff *= 15;
|
||||
score -= SentenceScoreDiff;
|
||||
for (String strmapTag : ITokenMapTag1.values()) {
|
||||
for (String strmapTag1 : ITokenMapTag2.values()) {
|
||||
if (strmapTag.equals(strmapTag1)) {
|
||||
score += 1450;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenForm1itr1 : strTokenForm1.values()) {
|
||||
for (String strTokenForm1itr2 : strTokenForm2.values()) {
|
||||
if (strTokenForm1itr1.equals(strTokenForm1itr2)) {
|
||||
score += 2600;
|
||||
} else if (strTokenForm1itr1.contains(strTokenForm1itr2)) {
|
||||
score += 500;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String strTokenStem : strTokenStems1.values()) {
|
||||
for (String strTokenStem1 : strTokenStems2.values()) {
|
||||
if (strTokenStem.equals(strTokenStem1)) {
|
||||
score += 1500;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) {
|
||||
score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650;
|
||||
}
|
||||
if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) {
|
||||
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
|
||||
}
|
||||
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
|
||||
score += (anotatorcounter1 - anotatorcounter2) * 400;
|
||||
}
|
||||
if (tokensCounter1 > 0 && tokensCounter2 > 0) {
|
||||
score += (tokensCounter1 + tokensCounter2) * 400;
|
||||
} else {
|
||||
int elseint = tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
||||
if (elseint > 0) {
|
||||
score -= elseint * 2;
|
||||
} else {
|
||||
score -= 1500;
|
||||
}
|
||||
}
|
||||
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
|
||||
double SentenceScoreDiff = leven.computeLevenshteinDistance();
|
||||
SentenceScoreDiff *= 15;
|
||||
score -= SentenceScoreDiff;
|
||||
} catch (Exception ex) {
|
||||
System.out.println("SENTIMENT stacktrace Overall catch: " + ex.getMessage() + "\n");
|
||||
}
|
||||
|
||||
@ -87,14 +87,6 @@ public class DiscordHandler {
|
||||
}
|
||||
}
|
||||
MessageResponseHandler.getMessage(strresult);
|
||||
new Thread(() -> {
|
||||
try {
|
||||
Datahandler.instance.checkIfUpdateStrings(false);
|
||||
Datahandler.instance.updateMatrixes();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}).start();
|
||||
}
|
||||
if (event.getMessage().getMentionedUsers().contains(api.getYourself())
|
||||
|| event.getServerTextChannel().get().toString().contains("general-autism")) {
|
||||
@ -105,6 +97,14 @@ public class DiscordHandler {
|
||||
System.out.print("\nResponseStr3: " + ResponseStr + "\n");
|
||||
event.getChannel().sendMessage(ResponseStr);
|
||||
}
|
||||
new Thread(() -> {
|
||||
try {
|
||||
Datahandler.instance.checkIfUpdateStrings(false);
|
||||
Datahandler.instance.updateMatrixes();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}).start();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user