UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

no more random numbers by names and no more repetitive data, good and end now i guess

Browse Source
This commit is contained in:
jenzur 2020-03-21 18:43:46 +01:00
parent ac56f386a3
commit abb033a9b8
4 changed files with 52 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
View File

@ -67,8 +67,7 @@ public class Datahandler {
private static Annotation strAnnoSentiment; private static Annotation strAnnoSentiment;
private static Annotation strAnnoJMWE; private static Annotation strAnnoJMWE;
private static CoreDocument coreDoc; private static CoreDocument coreDoc;
private static final ConcurrentMap<Integer, String> stringCache = new MapMaker().concurrencyLevel(2).makeMap(); private static final ConcurrentMap<Integer, String> stringCache = new MapMaker().concurrencyLevel(6).makeMap();
private static int RelationCounter = 0;
private static ConcurrentMap<String, Annotation> pipelineAnnotationCache; private static ConcurrentMap<String, Annotation> pipelineAnnotationCache;
private static ConcurrentMap<String, Annotation> pipelineSentimentAnnotationCache; private static ConcurrentMap<String, Annotation> pipelineSentimentAnnotationCache;
private static ConcurrentMap<String, Annotation> jmweAnnotationCache; private static ConcurrentMap<String, Annotation> jmweAnnotationCache;
@ -221,7 +220,7 @@ public class Datahandler {
hlStatsMessages.put(str, hlStatsMessages.size()); hlStatsMessages.put(str, hlStatsMessages.size());
} }
} }
int capacity = 500; int capacity = 1250;
hlStatsMessages.keySet().forEach(str -> { hlStatsMessages.keySet().forEach(str -> {
if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) { if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) {
String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null); String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null);
@ -271,29 +270,16 @@ public class Datahandler {
} }
} }
private ConcurrentMap<Integer, String> futuresReturnOverallEvaluation(List<SimilarityMatrix> similarityMatrixes, private ConcurrentMap<Integer, String> futuresReturnOverallEvaluation(List<SimilarityMatrix> similarityMatrixes) {
ConcurrentMap<Integer, String> strmapreturn) { ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(6).makeMap();
if (!similarityMatrixes.isEmpty()) { if (!similarityMatrixes.isEmpty()) {
String newPrimary = similarityMatrixes.get(0).getPrimaryString(); String newPrimary = similarityMatrixes.get(0).getPrimaryString();
int evaluationCap = 50000; int evaluationCap = 500;
boolean hitCap = false;
int iterator = 0; int iterator = 0;
for (SimilarityMatrix SMX : similarityMatrixes) { for (SimilarityMatrix SMX : similarityMatrixes) {
if (!newPrimary.equals(SMX.getPrimaryString())) { final Double scoreRelationNewMsgToRecentMsg = SMX.getDistance();
newPrimary = SMX.getPrimaryString(); if (scoreRelationNewMsgToRecentMsg > evaluationCap) {
strmapreturn = addSMXToMapReturn(strmapreturn, SMX); strmapreturn = addSMXToMapReturn(strmapreturn, SMX);
hitCap = false;
}
if (!hitCap) {
final Double scoreRelationNewMsgToRecentMsg = SMX.getDistance();
RelationCounter += scoreRelationNewMsgToRecentMsg > 500 ? scoreRelationNewMsgToRecentMsg * 2 : scoreRelationNewMsgToRecentMsg;
if (RelationCounter > evaluationCap) {
strmapreturn = addSMXToMapReturn(strmapreturn, SMX);
hitCap = true;
} else if (RelationCounter < evaluationCap * -1) {
addSMXToMapReturn(strmapreturn, SMX);
hitCap = true;
}
} }
//System.out.println("similarityMatrixes size: " + similarityMatrixes.size() + "\niterator: " + iterator); //System.out.println("similarityMatrixes size: " + similarityMatrixes.size() + "\niterator: " + iterator);
iterator++; iterator++;
@ -303,26 +289,23 @@ public class Datahandler {
} }
private ConcurrentMap<Integer, String> addSMXToMapReturn(ConcurrentMap<Integer, String> strmapreturn, SimilarityMatrix SMX) { private ConcurrentMap<Integer, String> addSMXToMapReturn(ConcurrentMap<Integer, String> strmapreturn, SimilarityMatrix SMX) {
//System.out.println("RelationCounter cap: " + RelationCounter); if (!strmapreturn.containsValue(SMX.getPrimaryString())) {
boolean related = RelationCounter > 500;
if (related) {
strmapreturn.put(strmapreturn.size(), SMX.getPrimaryString()); strmapreturn.put(strmapreturn.size(), SMX.getPrimaryString());
String transmittedStr = SMX.getSecondaryString(); String transmittedStr = SMX.getSecondaryString();
SentimentValueCache cacheValue1 = SMX.getCacheValue1(); SentimentValueCache cacheValue1 = SMX.getCacheValue1();
SentimentValueCache cacheValue2 = SMX.getCacheValue2(); SentimentValueCache cacheValue2 = SMX.getCacheValue2();
if (cacheValue1 != null && !sentimentCachingMap.keySet().contains(SMX.getPrimaryString())) { if (cacheValue1 != null && !sentimentCachingMap.keySet().contains(SMX.getPrimaryString())) {
sentimentCachingMap.put(SMX.getPrimaryString(), SMX.getCacheValue1()); sentimentCachingMap.put(SMX.getSecondaryString(), SMX.getCacheValue1());
} }
if (cacheValue2 != null && !sentimentCachingMap.keySet().contains(transmittedStr)) { if (cacheValue2 != null && !sentimentCachingMap.keySet().contains(transmittedStr)) {
sentimentCachingMap.put(transmittedStr, SMX.getCacheValue2()); sentimentCachingMap.put(transmittedStr, SMX.getCacheValue2());
} }
} }
RelationCounter = 0;
return strmapreturn; return strmapreturn;
} }
private List<SimilarityMatrix> StrComparringNoSentenceRelationMap( private List<SimilarityMatrix> StrComparringNoSentenceRelationMap(
ConcurrentMap<Integer, String> strCacheLocal, String str, ConcurrentMap<String, Annotation> localJMWEMap, ConcurrentMap<Integer, String> strCacheLocal, Collection<String> str, ConcurrentMap<String, Annotation> localJMWEMap,
ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation, ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
ConcurrentMap<String, CoreDocument> localCoreDocumentMap, CompletionService<SimilarityMatrix> ecs, int index) { ConcurrentMap<String, CoreDocument> localCoreDocumentMap, CompletionService<SimilarityMatrix> ecs, int index) {
int prefix_size = 125; int prefix_size = 125;
@ -342,35 +325,40 @@ public class Datahandler {
iterated++; iterated++;
} }
for (String str1 : randomIterationComparision) { for (String str1 : randomIterationComparision) {
if (!str.equals(str1)) { for (String str2 : str) {
SimilarityMatrix SMXInit = new SimilarityMatrix(str, str1); if (!str2.equals(str1)) {
SentimentValueCache sentimentCacheStr1 = sentimentCachingMap.getOrDefault(str1, null); SimilarityMatrix SMXInit = new SimilarityMatrix(str2, str1);
Callable<SimilarityMatrix> worker; SentimentValueCache sentimentCacheStr1 = sentimentCachingMap.getOrDefault(str1, null);
if (stringCache.size() < prefix_size) { Callable<SimilarityMatrix> worker;
worker = new SentimentAnalyzerTest(str, str1, SMXInit, if (stringCache.size() < prefix_size) {
localJMWEMap.get(str), localJMWEMap.get(str1), localPipelineAnnotation.get(str), worker = new SentimentAnalyzerTest(str2, str1, SMXInit,
localPipelineAnnotation.get(str1), localPipelineSentimentAnnotation.get(str), localJMWEMap.get(str), localJMWEMap.get(str1), localPipelineAnnotation.get(str),
localPipelineSentimentAnnotation.get(str1), localCoreDocumentMap.get(str), localCoreDocumentMap.get(str1), sentimentCacheStr, sentimentCacheStr1); localPipelineAnnotation.get(str1), localPipelineSentimentAnnotation.get(str),
} else { localPipelineSentimentAnnotation.get(str1), localCoreDocumentMap.get(str), localCoreDocumentMap.get(str1), sentimentCacheStr, sentimentCacheStr1);
worker = new SentimentAnalyzerTest(str, str1, SMXInit, } else {
localJMWEMap.get(str), jmweAnnotationCache.get(str1), localPipelineAnnotation.get(str), worker = new SentimentAnalyzerTest(str2, str1, SMXInit,
pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str), localJMWEMap.get(str), jmweAnnotationCache.get(str1), localPipelineAnnotation.get(str),
pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1), sentimentCacheStr, sentimentCacheStr1); pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str),
pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1), sentimentCacheStr, sentimentCacheStr1);
}
ecs.submit(worker);
index++;
} }
ecs.submit(worker);
index++;
} }
} }
for (int i = 0; i < index; i++) { for (int i = 0; i < index; i++) {
try { try {
Future<SimilarityMatrix> take = ecs.take(); Future<SimilarityMatrix> take = ecs.take();
SimilarityMatrix smx = take.get(); SimilarityMatrix smx = take.get();
smxReturnList.add(smx); if (smx != null && !smxReturnList.contains(smx)) {
smxReturnList.add(smx);
}
} catch (InterruptedException | ExecutionException ex) { } catch (InterruptedException | ExecutionException ex) {
// //
} }
} }
index = 0; index = 0;
System.out.println("smxReturnList size: " + smxReturnList.size());
return smxReturnList; return smxReturnList;
} }
@ -378,21 +366,17 @@ public class Datahandler {
ConcurrentMap<Integer, String> strCacheLocal, ConcurrentMap<String, Annotation> localJMWEMap, ConcurrentMap<Integer, String> strCacheLocal, ConcurrentMap<String, Annotation> localJMWEMap,
ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation, ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
ConcurrentMap<String, CoreDocument> localCoreDocumentMap) { ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(6).makeMap();
List<SimilarityMatrix> strSenseRelationMap = new ArrayList();
ExecutorService threadPool = Executors.newCachedThreadPool(); ExecutorService threadPool = Executors.newCachedThreadPool();
CompletionService<SimilarityMatrix> ecs = new ExecutorCompletionService<>(threadPool); CompletionService<SimilarityMatrix> ecs = new ExecutorCompletionService<>(threadPool);
int index = 0; int index = 0;
strmap.values().forEach(str -> { //System.out.println("strmap siuze: " + strmap.size());
List<SimilarityMatrix> localNoSentenceRelationList = StrComparringNoSentenceRelationMap(strCacheLocal, str, List<SimilarityMatrix> StrComparringNoSentenceRelationMap = StrComparringNoSentenceRelationMap(strCacheLocal, strmap.values(),
localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, ecs, index); localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, ecs, index);
for (SimilarityMatrix SMX : localNoSentenceRelationList) {
strSenseRelationMap.add(SMX);
}
});
threadPool.shutdown(); threadPool.shutdown();
Collections.sort(strSenseRelationMap, (e1, e2) -> e1.getPrimaryString().compareTo(e2.getPrimaryString())); //System.out.println("StrComparringNoSentenceRelationMap size: " + StrComparringNoSentenceRelationMap.size());
strmapreturn = futuresReturnOverallEvaluation(strSenseRelationMap, strmapreturn); Collections.sort(StrComparringNoSentenceRelationMap, (e1, e2) -> e1.getPrimaryString().compareTo(e2.getPrimaryString()));
ConcurrentMap<Integer, String> strmapreturn = futuresReturnOverallEvaluation(StrComparringNoSentenceRelationMap);
//System.out.println("strmapreturn size: " + strmapreturn.size());
return strmapreturn; return strmapreturn;
} }
@ -408,13 +392,15 @@ public class Datahandler {
public synchronized void checkIfUpdateStrings(boolean hlStatsMsg) throws CustomError { public synchronized void checkIfUpdateStrings(boolean hlStatsMsg) throws CustomError {
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) { if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr(); ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
System.out.println("str size: " + str.size());
str = cutContent(str, hlStatsMsg); str = cutContent(str, hlStatsMsg);
str = filterContent(str); str = filterContent(str);
str = removeSlacks(str); str = removeSlacks(str);
//System.out.println("finished removeSlacks \n" + str.size() + "\n"); //System.out.println("finished removeSlacks \n" + str.size() + "\n");
str = removeNonSensicalStrings(str); str = removeNonSensicalStrings(str);
System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n");
str = annotationCacheUpdate(str); str = annotationCacheUpdate(str);
//System.out.println("annotationCacheUpdate str size POST: " + str.size() + "\n"); System.out.println("annotationCacheUpdate str size POST: " + str.size() + "\n");
ConcurrentMap<Integer, String> strf = str; ConcurrentMap<Integer, String> strf = str;
if (!stringCache.isEmpty()) { if (!stringCache.isEmpty()) {
new Thread(() -> { new Thread(() -> {

9
ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java
View File

@ -20,16 +20,13 @@ import org.javacord.api.event.message.MessageCreateEvent;
*/ */
public class DoStuff { public class DoStuff {
public volatile boolean occupied = false; public static boolean occupied = false;
public boolean isOccupied() { public static boolean isOccupied() {
return occupied; return occupied;
} }
public DoStuff() { public static void doStuff(MessageCreateEvent event, DiscordApi api) {
}
public void doStuff(MessageCreateEvent event, DiscordApi api) {
if (!event.getMessage().getAuthor().isYourself()) { if (!event.getMessage().getAuthor().isYourself()) {
occupied = true; occupied = true;
String strtest = event.getServerTextChannel().get().getCategory().toString(); String strtest = event.getServerTextChannel().get().getCategory().toString();

15
ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java
View File

@ -78,18 +78,13 @@ public class MessageResponseHandler {
boolean isMatched = matcher.matches(); boolean isMatched = matcher.matches();
if (!emText.equals(personName) && !isMatched) { if (!emText.equals(personName) && !isMatched) {
for (CoreEntityMention emLastMsg : pipelineCoreDcoumentLastMsg.entityMentions()) { for (CoreEntityMention emLastMsg : pipelineCoreDcoumentLastMsg.entityMentions()) {
if (!emText.equals(emLastMsg.text())) { if (!emText.equals(emLastMsg.text()) && !Character.isDigit(emLastMsg.text().trim().charAt(0))) {
str = strreturn.replaceFirst(emText, emLastMsg.text()); //System.out.println("emLastMsg.text(): " + emLastMsg.text());
str = strreturn.substring(0, strreturn.indexOf(emText)) + " "
+ emLastMsg + " " + strreturn.substring(strreturn.indexOf(emText));
} }
} }
try { str += " " + personName;
//System.out.println("personName: " + personName + " str: " + str);
str = str.replace(str.substring(str.lastIndexOf(emText), str.length()), personName);
//System.out.println("str after: " + str);
} catch (Exception e) {
//System.out.println("failed replacing: " + e.getLocalizedMessage() + "\n");
str += " " + personName;
}
return str; return str;
} }
} }

5
ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java
View File

@ -48,10 +48,9 @@ public class DiscordHandler {
Datahandler.instance.updateStringCache(); Datahandler.instance.updateStringCache();
String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A"; String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A";
DiscordApi api = new DiscordApiBuilder().setToken(token).login().join(); DiscordApi api = new DiscordApiBuilder().setToken(token).login().join();
DoStuff dostuff = new DoStuff();
api.addMessageCreateListener(event -> { api.addMessageCreateListener(event -> {
if (!dostuff.isOccupied()) { if (!FunctionLayer.DoStuff.isOccupied()) {
dostuff.doStuff(event, api); FunctionLayer.DoStuff.doStuff(event, api);
} }
}); });
} }