UNLOZE/projects-jenz
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

attempted improving multithread conditions

Browse Source
This commit is contained in:
jenzur 2019-03-05 13:27:33 +01:00
parent d8d415cbe0
commit 7f154f3456
3 changed files with 101 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java
View File

@ -25,6 +25,7 @@ import edu.stanford.nlp.trees.Tree;
import edu.stanford.nlp.trees.TreebankLanguagePack; import edu.stanford.nlp.trees.TreebankLanguagePack;
import java.io.IOException; import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
import java.lang.management.ManagementFactory;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
@ -40,6 +41,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.logging.Level; import java.util.logging.Level;
@ -55,8 +57,9 @@ public class MYSQLDatahandler {
public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS); public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS);
public static MYSQLDatahandler instance = new MYSQLDatahandler(); public static MYSQLDatahandler instance = new MYSQLDatahandler();
public static int semeticsUpdateCount; public static int semeticsUpdateCount;
public static int threadCounter = 0; private static int counter = 0;
private volatile boolean refreshMatrixFromDB; private volatile boolean refreshMatrixFromDB;
private volatile boolean permitted = false;
private final ConcurrentMap<Integer, String> stringCache; private final ConcurrentMap<Integer, String> stringCache;
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap(); private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
private ConcurrentMap<Integer, String> multiprocessCalculations; private ConcurrentMap<Integer, String> multiprocessCalculations;
@ -79,7 +82,6 @@ public class MYSQLDatahandler {
private static AbstractSequenceClassifier<CoreLabel> classifier; private static AbstractSequenceClassifier<CoreLabel> classifier;
private static StanfordCoreNLP pipeline; private static StanfordCoreNLP pipeline;
private static StanfordCoreNLP pipelineSentiment; private static StanfordCoreNLP pipelineSentiment;
private static volatile boolean permitted = false;
public static AbstractSequenceClassifier<CoreLabel> getClassifier() { public static AbstractSequenceClassifier<CoreLabel> getClassifier() {
return classifier; return classifier;
@ -172,19 +174,18 @@ public class MYSQLDatahandler {
public synchronized void checkIfUpdateMatrixes() { public synchronized void checkIfUpdateMatrixes() {
refreshMatrixFromDB = false; refreshMatrixFromDB = false;
int calculationBoundaries = 6; int calculationBoundaries = 10;
int updateBadgesInteger = 80; int updateBadgesInteger = 250;
while (lHMSMX.size() < (stringCache.values().size() * stringCache.values().size()) - stringCache.values().size()) { while (lHMSMX.size() < (stringCache.values().size() * stringCache.values().size()) - stringCache.values().size()) {
if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) { if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) {
refreshMatrixFromDB = true; refreshMatrixFromDB = true;
if (threadCounter == 0) { lHMSMX = DataMapper.getAllRelationScores();
lHMSMX = DataMapper.getAllRelationScores(); stopwatch1.reset();
stopwatch1.reset();
}
} }
//requiring atleast 10 entries ensures no issues in case of empty stringcache
if (stringCache.values().size() > 10 && !refreshMatrixFromDB) { if (stringCache.values().size() > 10 && !refreshMatrixFromDB) {
if (threadCounter < 25) { if (counter <= 5) {
threadCounter++; counter++;
ConcurrentMap<Integer, String> stringCachelocal = stringCache; ConcurrentMap<Integer, String> stringCachelocal = stringCache;
List<Integer> updateLocal = updatedRows; List<Integer> updateLocal = updatedRows;
int random = -1; int random = -1;
@ -206,7 +207,7 @@ public class MYSQLDatahandler {
updatedRows.add(random); updatedRows.add(random);
semeticsUpdateCount = random; semeticsUpdateCount = random;
int beginindex = semeticsUpdateCount; int beginindex = semeticsUpdateCount;
semeticsUpdateCount += calculationBoundaries / 2; semeticsUpdateCount += calculationBoundaries;
int temp = semeticsUpdateCount; int temp = semeticsUpdateCount;
System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n"); System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n");
ConcurrentMap<Integer, String> strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, String> strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap();
@ -217,86 +218,87 @@ public class MYSQLDatahandler {
multiprocessCalculations.put(multiprocessCalculations.size() + 1, get); multiprocessCalculations.put(multiprocessCalculations.size() + 1, get);
ij++; ij++;
} }
new Thread(() -> { LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX; ConcurrentMap<Integer, String> strIndexAll = stringCachelocal;
ConcurrentMap<Integer, String> strIndexAll = stringCachelocal; ConcurrentMap<Integer, String> strIndexNavigatorL = strIndexNavigator;
ConcurrentMap<Integer, String> strIndexNavigatorL = strIndexNavigator; ConcurrentMap<Integer, String> randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, String> randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap(); int indexes = updateBadgesInteger;
int indexes = updateBadgesInteger; if (indexes >= strIndexAll.size()) {
if (indexes >= strIndexAll.size()) { indexes = strIndexAll.size() - 1;
indexes = strIndexAll.size() - 1; }
} int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes);
int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes); int ij1 = 0;
int ij1 = 0; while (beginindexes + ij1 < beginindexes + indexes) {
while (beginindexes + ij1 < beginindexes + indexes) { String get1 = strIndexAll.get(beginindexes + ij1);
String get1 = strIndexAll.get(beginindexes + ij1); randomIndexesToUpdate.put(ij1, get1);
randomIndexesToUpdate.put(ij1, get1); ij1++;
ij1++; }
} ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap(); System.out.println("activecount: " + java.lang.Thread.activeCount() + "\nThreadCount: " + ManagementFactory.getThreadMXBean().getThreadCount()
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + "\navailableprocessors: " + Runtime.getRuntime().availableProcessors() + "\n");
strIndexNavigatorL.values().forEach((str) -> { ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(),
randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { ForkJoinPool.defaultForkJoinWorkerThreadFactory,
boolean present = false; null, true);
if (multiprocessCalculations.values().contains(str1)) { strIndexNavigatorL.values().forEach((str) -> {
randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> {
boolean present = false;
if (multiprocessCalculations.values().contains(str1)) {
present = true;
} else if (LHMSMXLocal.containsKey(str)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str);
if (orDefault.containsKey(str1)) {
present = true; present = true;
} else if (LHMSMXLocal.containsKey(str)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str);
if (orDefault.containsKey(str1)) {
present = true;
}
} else if (LHMSMXLocal.containsKey(str1)) {
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str1);
if (orDefault.containsKey(str)) {
present = true;
}
} }
if (!present) { } else if (LHMSMXLocal.containsKey(str1)) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1); LinkedHashMap<String, Double> orDefault = LHMSMXLocal.get(str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX); if (orDefault.containsKey(str)) {
futures.put(futures.size() + 1, executor.submit(worker)); present = true;
} }
}); }
if (!present) {
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
futures.put(futures.size() + 1, executor.submit(worker));
}
}); });
executor.shutdown(); });
try { executor.shutdown();
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); try {
for (Future<SimilarityMatrix> future : futures.values()) { System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
SimilarityMatrix SMX = future.get(); for (Future<SimilarityMatrix> future : futures.values()) {
LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); SimilarityMatrix SMX = future.get();
if (get == null) { LinkedHashMap<String, Double> get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
get = new LinkedHashMap(); if (get == null) {
} get = new LinkedHashMap();
get.put(SMX.getSecondaryString(), SMX.getDistance());
lHMSMX.put(SMX.getPrimaryString(), get);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
} }
} catch (InterruptedException | ExecutionException ex) { get.put(SMX.getSecondaryString(), SMX.getDistance());
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); lHMSMX.put(SMX.getPrimaryString(), get);
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
} }
new Thread(() -> { } catch (InterruptedException | ExecutionException ex) {
try { Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
if (!matrixUpdateList.isEmpty()) { }
DataMapper.insertSementicMatrixes(matrixUpdateList); new Thread(() -> {
System.out.println("finished datamapper semetic insert"); try {
} if (!matrixUpdateList.isEmpty()) {
threadCounter--; DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("\nthreadCounter: " + threadCounter + "\n"); System.out.println("finished datamapper semetic insert");
} catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
} }
}).start(); } catch (CustomError ex) {
}). Logger.getLogger(MYSQLDatahandler.class
start(); .getName()).log(Level.SEVERE, null, ex);
}
}).start();
} else if (!permitted) { } else if (!permitted) {
permitted = true; permitted = true;
new Thread(() -> { new Thread(() -> {
ConcurrentMap<Integer, String> stringCachelocal = stringCache; ConcurrentMap<Integer, String> stringCachelocal = stringCache;
ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, SimilarityMatrix> matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap();
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(),
ForkJoinPool.defaultForkJoinWorkerThreadFactory,
null, true);
Collection<String> values = multiprocessCalculations.values(); Collection<String> values = multiprocessCalculations.values();
LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX; LinkedHashMap<String, LinkedHashMap<String, Double>> LHMSMXLocal = lHMSMX;
values.forEach((str) -> { values.forEach((str) -> {
@ -339,13 +341,13 @@ public class MYSQLDatahandler {
new Thread(() -> { new Thread(() -> {
try { try {
multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap();
updatedRows = new ArrayList();
counter = 0;
permitted = false; permitted = false;
if (!matrixUpdateList.isEmpty()) { if (!matrixUpdateList.isEmpty()) {
DataMapper.insertSementicMatrixes(matrixUpdateList); DataMapper.insertSementicMatrixes(matrixUpdateList);
System.out.println("finished datamapper semetic insert"); System.out.println("finished datamapper semetic insert");
} }
threadCounter--;
System.out.println("\nthreadCounter: " + threadCounter + "\n");
} catch (CustomError ex) { } catch (CustomError ex) {
Logger.getLogger(MYSQLDatahandler.class Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex); .getName()).log(Level.SEVERE, null, ex);
@ -432,7 +434,9 @@ public class MYSQLDatahandler {
System.out.println("none within 8 range"); System.out.println("none within 8 range");
ConcurrentMap<Integer, String> strCache = stringCache; ConcurrentMap<Integer, String> strCache = stringCache;
ConcurrentMap<Integer, Future<SimilarityMatrix>> futureslocal = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, Future<SimilarityMatrix>> futureslocal = new MapMaker().concurrencyLevel(2).makeMap();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(),
ForkJoinPool.defaultForkJoinWorkerThreadFactory,
null, true);
for (String str1 : strCache.values()) { for (String str1 : strCache.values()) {
if (!str.equals(str1)) { if (!str.equals(str1)) {
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, new SimilarityMatrix(str, str1)); Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, new SimilarityMatrix(str, str1));
@ -465,7 +469,9 @@ public class MYSQLDatahandler {
int minDistance = 8; int minDistance = 8;
String similar = ""; String similar = "";
ConcurrentMap<Integer, Future<DistanceObject>> futures = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, Future<DistanceObject>> futures = new MapMaker().concurrencyLevel(2).makeMap();
ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(),
ForkJoinPool.defaultForkJoinWorkerThreadFactory,
null, true);
concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> { concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> {
futures.put(futures.size() + 1, executor.submit(worker)); futures.put(futures.size() + 1, executor.submit(worker));
}); });
@ -484,10 +490,12 @@ public class MYSQLDatahandler {
} }
} catch (NullPointerException ex) { } catch (NullPointerException ex) {
System.out.println("failed future\n"); System.out.println("failed future\n");
} }
} }
} catch (InterruptedException | ExecutionException ex) { } catch (InterruptedException | ExecutionException ex) {
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(MYSQLDatahandler.class
.getName()).log(Level.SEVERE, null, ex);
} }
return similar; return similar;
} }

9
ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java
View File

@ -41,9 +41,14 @@ public class MessageResponseHandler {
public static String selectReponseMessage(String toString) throws CustomError { public static String selectReponseMessage(String toString) throws CustomError {
ConcurrentMap<Integer, String> str1 = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap<Integer, String> str1 = new MapMaker().concurrencyLevel(2).makeMap();
str1.put(str1.size() +1 , toString); str1.put(str1.size() + 1, toString);
str1 = MYSQLDatahandler.cutContent(str1); str1 = MYSQLDatahandler.cutContent(str1);
String strreturn = str1.get(0); String strreturn = "";
for (String str : str1.values()) {
if (!str.isEmpty()) {
strreturn = str;
}
}
String getResponseMsg = MYSQLDatahandler.instance.getResponseMsg(strreturn); String getResponseMsg = MYSQLDatahandler.instance.getResponseMsg(strreturn);
return getResponseMsg; return getResponseMsg;
} }

4
ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java
View File

@ -82,7 +82,9 @@ public class DiscordHandler {
MessageResponseHandler.getMessage(strresult); MessageResponseHandler.getMessage(strresult);
try { try {
MYSQLDatahandler.instance.checkIfUpdateStrings(); MYSQLDatahandler.instance.checkIfUpdateStrings();
MYSQLDatahandler.instance.checkIfUpdateMatrixes(); new Thread(() -> {
MYSQLDatahandler.instance.checkIfUpdateMatrixes();
}).start();
} catch (CustomError ex) { } catch (CustomError ex) {
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
} }