diff --git a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java index 17cf75a8..420a38bd 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java @@ -25,7 +25,6 @@ import edu.stanford.nlp.trees.Tree; import edu.stanford.nlp.trees.TreebankLanguagePack; import java.io.IOException; import java.io.StringReader; -import java.lang.management.ManagementFactory; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; @@ -40,7 +39,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -59,7 +57,6 @@ public class MYSQLDatahandler { public static int semeticsUpdateCount; private static int counter = 0; private volatile boolean refreshMatrixFromDB; - private volatile boolean permitted = false; private final ConcurrentMap stringCache; private LinkedHashMap> lHMSMX = new LinkedHashMap(); private ConcurrentMap multiprocessCalculations; @@ -80,6 +77,7 @@ public class MYSQLDatahandler { private static LexicalizedParser lp; private static TreebankLanguagePack tlp; private static AbstractSequenceClassifier classifier; + private ExecutorService executor; private static StanfordCoreNLP pipeline; private static StanfordCoreNLP pipelineSentiment; @@ -91,6 +89,12 @@ public class MYSQLDatahandler { MYSQLDatahandler.classifier = classifier; } + public void instantiateExecutor() { + this.executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), + ForkJoinPool.defaultForkJoinWorkerThreadFactory, + null, true); + } + public MYSQLDatahandler() { this.stopwatch = Stopwatch.createUnstarted(); this.stopwatch1 = Stopwatch.createStarted(); @@ -161,6 +165,14 @@ public class MYSQLDatahandler { return LHM; } + public int getlHMSMXSize() { + return lHMSMX.size(); + } + + public int getstringCacheSize() { + return stringCache.size(); + } + public void initiateMYSQL() throws SQLException, IOException { try { DataMapper.createTables(); @@ -174,187 +186,171 @@ public class MYSQLDatahandler { public synchronized void checkIfUpdateMatrixes() { refreshMatrixFromDB = false; - int calculationBoundaries = 10; + int calculationBoundaries = 5; int updateBadgesInteger = 250; - while (lHMSMX.size() < (stringCache.values().size() * stringCache.values().size()) - stringCache.values().size()) { - if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) { - refreshMatrixFromDB = true; - lHMSMX = DataMapper.getAllRelationScores(); - stopwatch1.reset(); - } - //requiring atleast 10 entries ensures no issues in case of empty stringcache - if (stringCache.values().size() > 10 && !refreshMatrixFromDB) { - if (counter <= 5) { - counter++; - ConcurrentMap stringCachelocal = stringCache; - List updateLocal = updatedRows; - int random = -1; - if (!updateLocal.contains(random)) { - updatedRows.add(random); - } - Collections.sort(updateLocal); - while (updateLocal.contains(random)) { - random = new Random().nextInt(stringCachelocal.values().size() - 6); - int indexPrev = Collections.binarySearch(updateLocal, random); - int indexNext = Collections.binarySearch(updateLocal, random + 6); - //-1 will always be index 0 - if (indexPrev > 0 && indexNext > 0) { - indexPrev = updateLocal.get(indexPrev); - indexNext = updateLocal.get(indexNext); - } - random = indexPrev < random - 5 && indexNext < random ? random : -1; - } + if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) { + refreshMatrixFromDB = true; + lHMSMX = DataMapper.getAllRelationScores(); + stopwatch1.reset(); + } + //requiring atleast 10 entries ensures no issues in case of empty stringcache + if (stringCache.values().size() > 10 && !refreshMatrixFromDB) { + if (counter <= 5) { + counter++; + ConcurrentMap stringCachelocal = stringCache; + List updateLocal = updatedRows; + int random = -1; + if (!updateLocal.contains(random)) { updatedRows.add(random); - semeticsUpdateCount = random; - int beginindex = semeticsUpdateCount; - semeticsUpdateCount += calculationBoundaries; - int temp = semeticsUpdateCount; - System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n"); - ConcurrentMap strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap(); - int ij = 0; - while (beginindex + ij < temp) { - String get = stringCachelocal.get(beginindex + ij); - strIndexNavigator.put(ij, get); - multiprocessCalculations.put(multiprocessCalculations.size() + 1, get); - ij++; - } - LinkedHashMap> LHMSMXLocal = lHMSMX; - ConcurrentMap strIndexAll = stringCachelocal; - ConcurrentMap strIndexNavigatorL = strIndexNavigator; - ConcurrentMap randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap(); - int indexes = updateBadgesInteger; - if (indexes >= strIndexAll.size()) { - indexes = strIndexAll.size() - 1; - } - int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes); - int ij1 = 0; - while (beginindexes + ij1 < beginindexes + indexes) { - String get1 = strIndexAll.get(beginindexes + ij1); - randomIndexesToUpdate.put(ij1, get1); - ij1++; - } - ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - System.out.println("activecount: " + java.lang.Thread.activeCount() + "\nThreadCount: " + ManagementFactory.getThreadMXBean().getThreadCount() - + "\navailableprocessors: " + Runtime.getRuntime().availableProcessors() + "\n"); - ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), - ForkJoinPool.defaultForkJoinWorkerThreadFactory, - null, true); - strIndexNavigatorL.values().forEach((str) -> { - randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { - boolean present = false; - if (multiprocessCalculations.values().contains(str1)) { - present = true; - } else if (LHMSMXLocal.containsKey(str)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str); - if (orDefault.containsKey(str1)) { - present = true; - } - } else if (LHMSMXLocal.containsKey(str1)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str1); - if (orDefault.containsKey(str)) { - present = true; - } - } - if (!present) { - SimilarityMatrix SMX = new SimilarityMatrix(str, str1); - Callable worker = new SentimentAnalyzerTest(str, str1, SMX); - futures.put(futures.size() + 1, executor.submit(worker)); - } - }); - }); - executor.shutdown(); - try { - System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); - for (Future future : futures.values()) { - SimilarityMatrix SMX = future.get(); - LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); - if (get == null) { - get = new LinkedHashMap(); - } - get.put(SMX.getSecondaryString(), SMX.getDistance()); - lHMSMX.put(SMX.getPrimaryString(), get); - matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); - } - } catch (InterruptedException | ExecutionException ex) { - Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); - } - new Thread(() -> { - try { - if (!matrixUpdateList.isEmpty()) { - DataMapper.insertSementicMatrixes(matrixUpdateList); - System.out.println("finished datamapper semetic insert"); - } - } catch (CustomError ex) { - Logger.getLogger(MYSQLDatahandler.class - .getName()).log(Level.SEVERE, null, ex); - } - }).start(); - } else if (!permitted) { - permitted = true; - new Thread(() -> { - ConcurrentMap stringCachelocal = stringCache; - ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), - ForkJoinPool.defaultForkJoinWorkerThreadFactory, - null, true); - Collection values = multiprocessCalculations.values(); - LinkedHashMap> LHMSMXLocal = lHMSMX; - values.forEach((str) -> { - stringCachelocal.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { - boolean present = false; - if (LHMSMXLocal.containsKey(str)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str); - if (orDefault.containsKey(str1)) { - present = true; - } - } else if (LHMSMXLocal.containsKey(str1)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str1); - if (orDefault.containsKey(str)) { - present = true; - } - } - if (!present) { - SimilarityMatrix SMX = new SimilarityMatrix(str, str1); - Callable worker = new SentimentAnalyzerTest(str, str1, SMX); - futures.put(futures.size() + 1, executor.submit(worker)); - } - }); - }); - executor.shutdown(); - try { - System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); - for (Future future : futures.values()) { - SimilarityMatrix SMX = future.get(); - LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); - if (get == null) { - get = new LinkedHashMap(); - } - get.put(SMX.getSecondaryString(), SMX.getDistance()); - lHMSMX.put(SMX.getPrimaryString(), get); - matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); - } - } catch (InterruptedException | ExecutionException ex) { - Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); - } - new Thread(() -> { - try { - multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); - updatedRows = new ArrayList(); - counter = 0; - permitted = false; - if (!matrixUpdateList.isEmpty()) { - DataMapper.insertSementicMatrixes(matrixUpdateList); - System.out.println("finished datamapper semetic insert"); - } - } catch (CustomError ex) { - Logger.getLogger(MYSQLDatahandler.class - .getName()).log(Level.SEVERE, null, ex); - } - }).start(); - }).start(); } + Collections.sort(updateLocal); + while (updateLocal.contains(random)) { + random = new Random().nextInt(stringCachelocal.values().size() - 6); + int indexPrev = Collections.binarySearch(updateLocal, random); + int indexNext = Collections.binarySearch(updateLocal, random + 6); + //-1 will always be index 0 + if (indexPrev > 0 && indexNext > 0) { + indexPrev = updateLocal.get(indexPrev); + indexNext = updateLocal.get(indexNext); + } + random = indexPrev < random - 5 && indexNext < random ? random : -1; + } + updatedRows.add(random); + semeticsUpdateCount = random; + int beginindex = semeticsUpdateCount; + semeticsUpdateCount += calculationBoundaries; + int temp = semeticsUpdateCount; + System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n"); + ConcurrentMap strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap(); + int ij = 0; + while (beginindex + ij < temp) { + String get = stringCachelocal.get(beginindex + ij); + strIndexNavigator.put(ij, get); + multiprocessCalculations.put(multiprocessCalculations.size() + 1, get); + ij++; + } + LinkedHashMap> LHMSMXLocal = lHMSMX; + ConcurrentMap strIndexAll = stringCachelocal; + ConcurrentMap strIndexNavigatorL = strIndexNavigator; + ConcurrentMap randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap(); + int indexes = updateBadgesInteger; + if (indexes >= strIndexAll.size()) { + indexes = strIndexAll.size() - 1; + } + int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes); + int ij1 = 0; + while (beginindexes + ij1 < beginindexes + indexes) { + String get1 = strIndexAll.get(beginindexes + ij1); + randomIndexesToUpdate.put(ij1, get1); + ij1++; + } + ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); + strIndexNavigatorL.values().forEach((str) -> { + randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { + boolean present = false; + if (multiprocessCalculations.values().contains(str1)) { + present = true; + } else if (LHMSMXLocal.containsKey(str)) { + LinkedHashMap orDefault = LHMSMXLocal.get(str); + if (orDefault.containsKey(str1)) { + present = true; + } + } else if (LHMSMXLocal.containsKey(str1)) { + LinkedHashMap orDefault = LHMSMXLocal.get(str1); + if (orDefault.containsKey(str)) { + present = true; + } + } + if (!present) { + SimilarityMatrix SMX = new SimilarityMatrix(str, str1); + Callable worker = new SentimentAnalyzerTest(str, str1, SMX); + futures.put(futures.size() + 1, executor.submit(worker)); + } + }); + }); + try { + System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); + for (Future future : futures.values()) { + SimilarityMatrix SMX = future.get(); + LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); + if (get == null) { + get = new LinkedHashMap(); + } + get.put(SMX.getSecondaryString(), SMX.getDistance()); + lHMSMX.put(SMX.getPrimaryString(), get); + matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); + } + } catch (InterruptedException | ExecutionException ex) { + Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + } + new Thread(() -> { + try { + if (!matrixUpdateList.isEmpty()) { + DataMapper.insertSementicMatrixes(matrixUpdateList); + System.out.println("finished datamapper semetic insert"); + } + } catch (CustomError ex) { + Logger.getLogger(MYSQLDatahandler.class + .getName()).log(Level.SEVERE, null, ex); + } + }).start(); + } else { + ConcurrentMap stringCachelocal = stringCache; + ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); + Collection values = multiprocessCalculations.values(); + LinkedHashMap> LHMSMXLocal = lHMSMX; + values.forEach((str) -> { + stringCachelocal.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { + boolean present = false; + if (LHMSMXLocal.containsKey(str)) { + LinkedHashMap orDefault = LHMSMXLocal.get(str); + if (orDefault.containsKey(str1)) { + present = true; + } + } else if (LHMSMXLocal.containsKey(str1)) { + LinkedHashMap orDefault = LHMSMXLocal.get(str1); + if (orDefault.containsKey(str)) { + present = true; + } + } + if (!present) { + SimilarityMatrix SMX = new SimilarityMatrix(str, str1); + Callable worker = new SentimentAnalyzerTest(str, str1, SMX); + futures.put(futures.size() + 1, executor.submit(worker)); + } + }); + }); + try { + System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); + for (Future future : futures.values()) { + SimilarityMatrix SMX = future.get(); + LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); + if (get == null) { + get = new LinkedHashMap(); + } + get.put(SMX.getSecondaryString(), SMX.getDistance()); + lHMSMX.put(SMX.getPrimaryString(), get); + matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); + } + } catch (InterruptedException | ExecutionException ex) { + Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + } + multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); + updatedRows = new ArrayList(); + counter = 0; + new Thread(() -> { + try { + if (!matrixUpdateList.isEmpty()) { + DataMapper.insertSementicMatrixes(matrixUpdateList); + System.out.println("finished datamapper semetic insert"); + } + } catch (CustomError ex) { + Logger.getLogger(MYSQLDatahandler.class + .getName()).log(Level.SEVERE, null, ex); + } + }).start(); } } } @@ -434,16 +430,12 @@ public class MYSQLDatahandler { System.out.println("none within 8 range"); ConcurrentMap strCache = stringCache; ConcurrentMap> futureslocal = new MapMaker().concurrencyLevel(2).makeMap(); - ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), - ForkJoinPool.defaultForkJoinWorkerThreadFactory, - null, true); for (String str1 : strCache.values()) { if (!str.equals(str1)) { Callable worker = new SentimentAnalyzerTest(str, str1, new SimilarityMatrix(str, str1)); futureslocal.put(futureslocal.size() + 1, executor.submit(worker)); } } - executor.shutdown(); int index = 0; for (Future future : futureslocal.values()) { try { @@ -469,13 +461,9 @@ public class MYSQLDatahandler { int minDistance = 8; String similar = ""; ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - ExecutorService executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), - ForkJoinPool.defaultForkJoinWorkerThreadFactory, - null, true); concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> { futures.put(futures.size() + 1, executor.submit(worker)); }); - executor.shutdown(); try { for (Future future : futures.values()) { DistanceObject d = future.get(); diff --git a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java index b162af71..df7bcd20 100644 --- a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java +++ b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java @@ -44,7 +44,11 @@ public class DiscordHandler { } }).start(); MYSQLDatahandler.shiftReduceParserInitiate(); - MYSQLDatahandler.instance.checkIfUpdateMatrixes(); + MYSQLDatahandler.instance.instantiateExecutor(); + while (MYSQLDatahandler.instance.getlHMSMXSize() < (MYSQLDatahandler.instance.getstringCacheSize() + * MYSQLDatahandler.instance.getstringCacheSize()) - MYSQLDatahandler.instance.getstringCacheSize()) { + MYSQLDatahandler.instance.checkIfUpdateMatrixes(); + } String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A"; DiscordApi api = new DiscordApiBuilder().setToken(token).login().join(); api.addMessageCreateListener(event -> {