UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

slight moving, predominantly calculation updates

Browse Source
This commit is contained in:
jenzur 2019-03-31 01:22:25 +01:00
parent feaf9b0adc
commit d788d76ac6
5 changed files with 503 additions and 367 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ArtificialAutism/src/main/java/DataLayer/DataMapper.java
View File

@ -122,7 +122,7 @@ public class DataMapper {
l_cCon = DBCPDataSource.getConnection();
l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY);
l_pStatement.setFetchSize(Integer.MIN_VALUE);
l_pStatement.setFetchSize(0);
System.out.println("Matrix update size: " + WS4JListUpdate.size());
for (SimilarityMatrix ws4j : WS4JListUpdate.values()) {
l_pStatement.setString(1, ws4j.getPrimaryString());
@ -146,8 +146,9 @@ public class DataMapper {
String l_sSQL = "SELECT * FROM `WordMatrix`";
try (PreparedStatement l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
java.sql.ResultSet.CONCUR_READ_ONLY)) {
l_pStatement.setFetchSize(Integer.MIN_VALUE);
l_pStatement.setFetchSize(0);
try (ResultSet l_rsSearch = l_pStatement.executeQuery()) {
l_rsSearch.setFetchSize(0);
int i = 0;
LinkedHashMap<String, Double> LHMLocal = new LinkedHashMap();
while (l_rsSearch.next()) {

15
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
View File

@ -217,7 +217,7 @@ public class Datahandler {
public void addHLstatsMessages() {
ConcurrentMap<Integer, String> hlStatsMessages = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> strCacheLocal = stringCache;
int hardcap = 8500;
int hardcap = 55000;
int ij = 0;
for (String str : DataMapper.getHLstatsMessages().values()) {
hlStatsMessages.put(ij, str);
@ -483,7 +483,7 @@ public class Datahandler {
public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings) {
similar = "";
minDistance = 7.5;
minDistance = 12.5;
concurrentStrings.values().parallelStream().forEach((str) -> {
LevenshteinDistance leven = new LevenshteinDistance(toBeCompared, str);
double distance = leven.computeLevenshteinDistance();
@ -630,8 +630,8 @@ public class Datahandler {
}
private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
ShiftReduceParser model = getModel();
MaxentTagger tagger = getTagger();
ShiftReduceParser modelLocal = getModel();
MaxentTagger taggerLocal = getTagger();
ConcurrentMap<Integer, String> strreturn = new MapMaker().concurrencyLevel(2).makeMap();
str.values().parallelStream().forEach(str1 -> {
ConcurrentMap<Integer, String> TGWList = new MapMaker().concurrencyLevel(2).makeMap();
@ -646,8 +646,8 @@ public class Datahandler {
for (List<HasWord> sentence : tokenizer) {
int counter = 0;
List<TaggedWord> taggedWords;
List<TaggedWord> tagged1 = tagger.tagSentence(sentence);
Tree tree = model.apply(tagged1);
List<TaggedWord> tagged1 = taggerLocal.tagSentence(sentence);
Tree tree = modelLocal.apply(tagged1);
taggedWords = tree.taggedYield();
for (TaggedWord TGW : taggedWords) {
if (!TGWList.values().contains(TGW.tag()) && !TGW.tag().equals(":") && !TGW.word().equals(TGW.tag())) {
@ -659,7 +659,6 @@ public class Datahandler {
ConcurrentMap<Integer, Word> wordList = new MapMaker().concurrencyLevel(2).makeMap();
for (Word lab : tree.yieldWords()) {
if (lab != null && lab.word() != null) {
//System.out.println("lab: " + lab + " \n");
if (!wordList.values().contains(lab) && lab.value() != null && !lab.value().equals(":")) {
wordList.put(wordList.size() + 1, lab);
addCounter++;
@ -681,7 +680,7 @@ public class Datahandler {
for (String strVals : values) {
LevenshteinDistance leven = new LevenshteinDistance(strVals, str1);
double Distance = leven.computeLevenshteinDistance();
int maxpermittedDistance = 2;
int maxpermittedDistance = 5;
if (Distance < maxpermittedDistance) {
tooclosematch = true;
break;

6
ArtificialAutism/src/main/java/FunctionLayer/LevenshteinDistance.java
View File

@ -5,12 +5,6 @@
*/
package FunctionLayer;
import com.google.common.collect.MapMaker;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
/**
*
* @author install1

168
ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java
View File

@ -38,11 +38,14 @@ import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.OptionalDouble;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BinaryOperator;
import java.util.function.Function;
import org.ejml.simple.SimpleMatrix;
/*
@ -228,13 +231,133 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree));
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
score += simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : dot > 0 ? 50 - dot : 50).map((subtracter) -> {
ConcurrentMap<Integer, Double> AccumulateDotMap = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Double> subtractorMap = new MapMaker().concurrencyLevel(2).makeMap();
score += simpleSMXlist.values().stream().map(new Function<SimpleMatrix, Double>() {
@Override
public Double apply(SimpleMatrix simpleSMX) {
return predictions.dot(simpleSMX) * 100;
}
}).map(new Function<Double, Double>() {
@Override
public Double apply(Double dot) {
AccumulateDotMap.put(AccumulateDotMap.size() + 1, dot);
return dot > 50 ? dot - 100 : dot > 0 ? 100 - dot : 0;
}
}).map((subtracter) -> {
subtractorMap.put(subtractorMap.size() + 1, subtracter);
subtracter *= 25; //25
return subtracter;
}).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator + _item);
}).map(new Function<Double, Double>() {
@Override
public Double apply(Double subtracter) {
return subtracter;
}
}).reduce(score, new BinaryOperator<Double>() {
@Override
public Double apply(Double accumulator, Double _item) {
int accumulator1 = 0;
while (accumulator < 0) {
accumulator++;
accumulator1++;
}
return accumulator1 + _item;
}
});
Double subTracPre = 0.0;
for (Double subtractors : subtractorMap.values()) {
if (Objects.equals(subTracPre, subtractors)) {
score -= 2000;
}
subTracPre = subtractors;
}
score += simpleSMXlist.values().stream().map(new Function<SimpleMatrix, Double>() {
@Override
public Double apply(SimpleMatrix simpleSMX) {
return simpleSMX.dot(predictions) * 100;
}
}).map(new Function<Double, Double>() {
@Override
public Double apply(Double dot) {
AccumulateDotMap.put(AccumulateDotMap.size() + 1, dot);
return dot > 50 ? dot - 50 : dot > 0 ? 50 - dot : 0;
}
}).map((subtracter) -> {
subtracter *= 25; //25
return subtracter;
}).map(new Function<Double, Double>() {
@Override
public Double apply(Double subtracter) {
return subtracter;
}
}).reduce(score, new BinaryOperator<Double>() {
@Override
public Double apply(Double accumulator, Double _item) {
int accumulator1 = 0;
while (accumulator < 0) {
accumulator++;
accumulator1++;
}
return accumulator1 + _item;
}
});
Double preAccumulatorDot = 0.0;
Double postAccumulatorDot = 0.0;
for (Double accumulators : AccumulateDotMap.values()) {
if (preAccumulatorDot == accumulators) {
if (postAccumulatorDot == accumulators) {
score -= 4000;
}
postAccumulatorDot = accumulators;
}
preAccumulatorDot = accumulators;
}
subTracPre = 0.0;
for (Double subtractors : subtractorMap.values()) {
if (Objects.equals(subTracPre, subtractors)) {
score -= 2000;
}
subTracPre = subtractors;
}
Double preDot = 0.0;
Double postDot = 0.0;
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
double dot = nodeVector.dot(simpleSMX);
double elementSum = nodeVector.kron(simpleSMX).elementSum();
if (preDot == dot) {
if (postDot == dot) {
score -= 4000;
}
postDot = dot;
}
preDot = dot;
elementSum = Math.round(elementSum * 100.0) / 100.0;
elementSumCounter.put(elementSumCounter.size() + 1, elementSum);
dotMap.put(dotMap.size() + 1, dot);
if (dot < 0.1) {
score += 256;
}
if (dot > 0.50) {
score -= 2400;
}
if (elementSum < 0.01 && elementSum > 0.00) {
score += 3300;
} else if (elementSum > 0.1 && elementSum < 0.2) {
score += 1100;
} else {
score -= elementSum * 1424;
}
}
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
double dot = simpleSMX.dot(nodeVector);
double elementSum = simpleSMX.kron(nodeVector).elementSum();
if (preDot == dot) {
if (postDot == dot) {
score -= 4000;
}
postDot = dot;
}
preDot = dot;
elementSum = Math.round(elementSum * 100.0) / 100.0;
elementSumCounter.put(elementSumCounter.size() + 1, elementSum);
dotMap.put(dotMap.size() + 1, dot);
@ -253,19 +376,32 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
}
}
}
if (dotMap.values().size() > 1) {
OptionalDouble minvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).min();
OptionalDouble maxvalueDots = dotMap.values().stream().mapToDouble(Double::doubleValue).max();
if (maxvalueDots.getAsDouble() - minvalueDots.getAsDouble() < 0.05) {
double total = minvalueDots.getAsDouble() + maxvalueDots.getAsDouble();
boolean permitted = false;
if (minvalueDots.getAsDouble() != maxvalueDots.getAsDouble()) {
permitted = true;
}
if (permitted) {
Double dotsVariance = maxvalueDots.getAsDouble() - minvalueDots.getAsDouble();
if (maxvalueDots.getAsDouble() > minvalueDots.getAsDouble() * 10) {
score -= 5500;
} else if (minvalueDots.getAsDouble() < -0.10) {
score -= 3500;
} else if (dotsVariance < 0.5) {
score += 3500;
} else if (dotsVariance > minvalueDots.getAsDouble() * 2) {
score += 3500;
}
}
if (elementSumCounter.values().size() > 1){
OptionalDouble minvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).min();
OptionalDouble maxvalueElements = elementSumCounter.values().stream().mapToDouble(Double::doubleValue).max();
if (maxvalueElements.getAsDouble() - minvalueElements.getAsDouble() < 0.05) {
Double elementsVariance = maxvalueElements.getAsDouble() - minvalueElements.getAsDouble();
if (elementsVariance < 0.05 && maxvalueElements.getAsDouble() > 0.0 && minvalueElements.getAsDouble() > 0.0 && elementsVariance > 0.000) {
score += 3500;
}
} else if (minvalueElements.getAsDouble() < 0.0 && minvalueElements.getAsDouble() - maxvalueElements.getAsDouble() < 0.50) {
score -= 2500;
}
score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500;
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
@ -299,18 +435,20 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
if (longest1 != longest2) {
long deffLongest = longest1 > longest2 ? longest1 : longest2;
long deffshorter = longest1 < longest2 ? longest1 : longest2;
//deffLongest >= (deffshorter * 2)
if (deffLongest < (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
if (deffLongest > deffshorter * 5) {
score -= 5500;
} else if (deffLongest < (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
score += (deffLongest - deffshorter) * 120;
} else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) {
score += (deffLongest - deffshorter) * 120;
} else if (deffLongest - deffshorter < 2) {
score += (deffLongest + deffshorter) * 40;
} else if (deffLongest - deffshorter <= 5){
score += 2500;
} else {
score -= (deffLongest - deffshorter) * 50;
}
if (deffLongest - deffshorter <= 5) {
score += 2500;
}
}
int tokensCounter1 = 0;
int tokensCounter2 = 0;
@ -456,13 +594,17 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
}
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
score += (anotatorcounter1 + anotatorcounter2) * 400;
score += (anotatorcounter1 - anotatorcounter2) * 400;
}
if (tokensCounter1 > 0 && tokensCounter2 > 0) {
score += (tokensCounter1 + tokensCounter2) * 400;
} else {
int elseint = tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
score -= elseint;
if (elseint > 0) {
score -= elseint * 2;
} else {
score -= 1500;
}
}
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
double SentenceScoreDiff = leven.computeLevenshteinDistance();

16
ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java
View File

@ -87,14 +87,6 @@ public class DiscordHandler {
}
}
MessageResponseHandler.getMessage(strresult);
new Thread(() -> {
try {
Datahandler.instance.checkIfUpdateStrings(false);
Datahandler.instance.updateMatrixes();
} catch (CustomError ex) {
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
}
}).start();
}
if (event.getMessage().getMentionedUsers().contains(api.getYourself())
|| event.getServerTextChannel().get().toString().contains("general-autism")) {
@ -105,6 +97,14 @@ public class DiscordHandler {
System.out.print("\nResponseStr3: " + ResponseStr + "\n");
event.getChannel().sendMessage(ResponseStr);
}
new Thread(() -> {
try {
Datahandler.instance.checkIfUpdateStrings(false);
Datahandler.instance.updateMatrixes();
} catch (CustomError ex) {
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
}
}).start();
} catch (CustomError ex) {
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
}