diff --git a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java index d797135a..17cf75a8 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java @@ -25,6 +25,7 @@ import edu.stanford.nlp.trees.Tree; import edu.stanford.nlp.trees.TreebankLanguagePack; import java.io.IOException; import java.io.StringReader; +import java.lang.management.ManagementFactory; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; @@ -40,6 +41,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinPool; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -55,8 +57,9 @@ public class MYSQLDatahandler { public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS); public static MYSQLDatahandler instance = new MYSQLDatahandler(); public static int semeticsUpdateCount; - public static int threadCounter = 0; + private static int counter = 0; private volatile boolean refreshMatrixFromDB; + private volatile boolean permitted = false; private final ConcurrentMap stringCache; private LinkedHashMap> lHMSMX = new LinkedHashMap(); private ConcurrentMap multiprocessCalculations; @@ -79,7 +82,6 @@ public class MYSQLDatahandler { private static AbstractSequenceClassifier classifier; private static StanfordCoreNLP pipeline; private static StanfordCoreNLP pipelineSentiment; - private static volatile boolean permitted = false; public static AbstractSequenceClassifier getClassifier() { return classifier; @@ -172,19 +174,18 @@ public class MYSQLDatahandler { public synchronized void checkIfUpdateMatrixes() { refreshMatrixFromDB = false; - int calculationBoundaries = 6; - int updateBadgesInteger = 80; + int calculationBoundaries = 10; + int updateBadgesInteger = 250; while (lHMSMX.size() < (stringCache.values().size() * stringCache.values().size()) - stringCache.values().size()) { if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) { refreshMatrixFromDB = true; - if (threadCounter == 0) { - lHMSMX = DataMapper.getAllRelationScores(); - stopwatch1.reset(); - } + lHMSMX = DataMapper.getAllRelationScores(); + stopwatch1.reset(); } + //requiring atleast 10 entries ensures no issues in case of empty stringcache if (stringCache.values().size() > 10 && !refreshMatrixFromDB) { - if (threadCounter < 25) { - threadCounter++; + if (counter <= 5) { + counter++; ConcurrentMap stringCachelocal = stringCache; List updateLocal = updatedRows; int random = -1; @@ -206,7 +207,7 @@ public class MYSQLDatahandler { updatedRows.add(random); semeticsUpdateCount = random; int beginindex = semeticsUpdateCount; - semeticsUpdateCount += calculationBoundaries / 2; + semeticsUpdateCount += calculationBoundaries; int temp = semeticsUpdateCount; System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n"); ConcurrentMap strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap(); @@ -217,86 +218,87 @@ public class MYSQLDatahandler { multiprocessCalculations.put(multiprocessCalculations.size() + 1, get); ij++; } - new Thread(() -> { - LinkedHashMap> LHMSMXLocal = lHMSMX; - ConcurrentMap strIndexAll = stringCachelocal; - ConcurrentMap strIndexNavigatorL = strIndexNavigator; - ConcurrentMap randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap(); - int indexes = updateBadgesInteger; - if (indexes >= strIndexAll.size()) { - indexes = strIndexAll.size() - 1; - } - int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes); - int ij1 = 0; - while (beginindexes + ij1 < beginindexes + indexes) { - String get1 = strIndexAll.get(beginindexes + ij1); - randomIndexesToUpdate.put(ij1, get1); - ij1++; - } - ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - strIndexNavigatorL.values().forEach((str) -> { - randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { - boolean present = false; - if (multiprocessCalculations.values().contains(str1)) { + LinkedHashMap> LHMSMXLocal = lHMSMX; + ConcurrentMap strIndexAll = stringCachelocal; + ConcurrentMap strIndexNavigatorL = strIndexNavigator; + ConcurrentMap randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap(); + int indexes = updateBadgesInteger; + if (indexes >= strIndexAll.size()) { + indexes = strIndexAll.size() - 1; + } + int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes); + int ij1 = 0; + while (beginindexes + ij1 < beginindexes + indexes) { + String get1 = strIndexAll.get(beginindexes + ij1); + randomIndexesToUpdate.put(ij1, get1); + ij1++; + } + ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); + System.out.println("activecount: " + java.lang.Thread.activeCount() + "\nThreadCount: " + ManagementFactory.getThreadMXBean().getThreadCount() + + "\navailableprocessors: " + Runtime.getRuntime().availableProcessors() + "\n"); + ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), + ForkJoinPool.defaultForkJoinWorkerThreadFactory, + null, true); + strIndexNavigatorL.values().forEach((str) -> { + randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { + boolean present = false; + if (multiprocessCalculations.values().contains(str1)) { + present = true; + } else if (LHMSMXLocal.containsKey(str)) { + LinkedHashMap orDefault = LHMSMXLocal.get(str); + if (orDefault.containsKey(str1)) { present = true; - } else if (LHMSMXLocal.containsKey(str)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str); - if (orDefault.containsKey(str1)) { - present = true; - } - } else if (LHMSMXLocal.containsKey(str1)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str1); - if (orDefault.containsKey(str)) { - present = true; - } } - if (!present) { - SimilarityMatrix SMX = new SimilarityMatrix(str, str1); - Callable worker = new SentimentAnalyzerTest(str, str1, SMX); - futures.put(futures.size() + 1, executor.submit(worker)); + } else if (LHMSMXLocal.containsKey(str1)) { + LinkedHashMap orDefault = LHMSMXLocal.get(str1); + if (orDefault.containsKey(str)) { + present = true; } - }); + } + if (!present) { + SimilarityMatrix SMX = new SimilarityMatrix(str, str1); + Callable worker = new SentimentAnalyzerTest(str, str1, SMX); + futures.put(futures.size() + 1, executor.submit(worker)); + } }); - executor.shutdown(); - try { - System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); - for (Future future : futures.values()) { - SimilarityMatrix SMX = future.get(); - LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); - if (get == null) { - get = new LinkedHashMap(); - } - get.put(SMX.getSecondaryString(), SMX.getDistance()); - lHMSMX.put(SMX.getPrimaryString(), get); - matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); + }); + executor.shutdown(); + try { + System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); + for (Future future : futures.values()) { + SimilarityMatrix SMX = future.get(); + LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); + if (get == null) { + get = new LinkedHashMap(); } - } catch (InterruptedException | ExecutionException ex) { - Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + get.put(SMX.getSecondaryString(), SMX.getDistance()); + lHMSMX.put(SMX.getPrimaryString(), get); + matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); } - new Thread(() -> { - try { - if (!matrixUpdateList.isEmpty()) { - DataMapper.insertSementicMatrixes(matrixUpdateList); - System.out.println("finished datamapper semetic insert"); - } - threadCounter--; - System.out.println("\nthreadCounter: " + threadCounter + "\n"); - } catch (CustomError ex) { - Logger.getLogger(MYSQLDatahandler.class - .getName()).log(Level.SEVERE, null, ex); + } catch (InterruptedException | ExecutionException ex) { + Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + } + new Thread(() -> { + try { + if (!matrixUpdateList.isEmpty()) { + DataMapper.insertSementicMatrixes(matrixUpdateList); + System.out.println("finished datamapper semetic insert"); } - }).start(); - }). - start(); + } catch (CustomError ex) { + Logger.getLogger(MYSQLDatahandler.class + .getName()).log(Level.SEVERE, null, ex); + } + }).start(); } else if (!permitted) { permitted = true; new Thread(() -> { ConcurrentMap stringCachelocal = stringCache; ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), + ForkJoinPool.defaultForkJoinWorkerThreadFactory, + null, true); Collection values = multiprocessCalculations.values(); LinkedHashMap> LHMSMXLocal = lHMSMX; values.forEach((str) -> { @@ -339,13 +341,13 @@ public class MYSQLDatahandler { new Thread(() -> { try { multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); + updatedRows = new ArrayList(); + counter = 0; permitted = false; if (!matrixUpdateList.isEmpty()) { DataMapper.insertSementicMatrixes(matrixUpdateList); System.out.println("finished datamapper semetic insert"); } - threadCounter--; - System.out.println("\nthreadCounter: " + threadCounter + "\n"); } catch (CustomError ex) { Logger.getLogger(MYSQLDatahandler.class .getName()).log(Level.SEVERE, null, ex); @@ -432,7 +434,9 @@ public class MYSQLDatahandler { System.out.println("none within 8 range"); ConcurrentMap strCache = stringCache; ConcurrentMap> futureslocal = new MapMaker().concurrencyLevel(2).makeMap(); - ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), + ForkJoinPool.defaultForkJoinWorkerThreadFactory, + null, true); for (String str1 : strCache.values()) { if (!str.equals(str1)) { Callable worker = new SentimentAnalyzerTest(str, str1, new SimilarityMatrix(str, str1)); @@ -465,7 +469,9 @@ public class MYSQLDatahandler { int minDistance = 8; String similar = ""; ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), + ForkJoinPool.defaultForkJoinWorkerThreadFactory, + null, true); concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> { futures.put(futures.size() + 1, executor.submit(worker)); }); @@ -484,10 +490,12 @@ public class MYSQLDatahandler { } } catch (NullPointerException ex) { System.out.println("failed future\n"); + } } } catch (InterruptedException | ExecutionException ex) { - Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + Logger.getLogger(MYSQLDatahandler.class + .getName()).log(Level.SEVERE, null, ex); } return similar; } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java b/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java index e21ea682..6583ed2c 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/MessageResponseHandler.java @@ -41,9 +41,14 @@ public class MessageResponseHandler { public static String selectReponseMessage(String toString) throws CustomError { ConcurrentMap str1 = new MapMaker().concurrencyLevel(2).makeMap(); - str1.put(str1.size() +1 , toString); + str1.put(str1.size() + 1, toString); str1 = MYSQLDatahandler.cutContent(str1); - String strreturn = str1.get(0); + String strreturn = ""; + for (String str : str1.values()) { + if (!str.isEmpty()) { + strreturn = str; + } + } String getResponseMsg = MYSQLDatahandler.instance.getResponseMsg(strreturn); return getResponseMsg; } diff --git a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java index 5964cff1..b162af71 100644 --- a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java +++ b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java @@ -82,7 +82,9 @@ public class DiscordHandler { MessageResponseHandler.getMessage(strresult); try { MYSQLDatahandler.instance.checkIfUpdateStrings(); - MYSQLDatahandler.instance.checkIfUpdateMatrixes(); + new Thread(() -> { + MYSQLDatahandler.instance.checkIfUpdateMatrixes(); + }).start(); } catch (CustomError ex) { Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex); }