From 05683ce4be85eecb517cbf997a16be3371d90af7 Mon Sep 17 00:00:00 2001 From: jenzur Date: Wed, 6 Mar 2019 23:15:02 +0100 Subject: [PATCH] simplifying matrix updating quite alot --- .../java/FunctionLayer/MYSQLDatahandler.java | 268 ++++++------------ .../PresentationLayer/DiscordHandler.java | 2 +- 2 files changed, 94 insertions(+), 176 deletions(-) diff --git a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java index 420a38bd..678424a8 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java @@ -35,6 +35,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Random; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; @@ -42,6 +43,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; @@ -55,12 +57,9 @@ public class MYSQLDatahandler { public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS); public static MYSQLDatahandler instance = new MYSQLDatahandler(); public static int semeticsUpdateCount; - private static int counter = 0; private volatile boolean refreshMatrixFromDB; private final ConcurrentMap stringCache; private LinkedHashMap> lHMSMX = new LinkedHashMap(); - private ConcurrentMap multiprocessCalculations; - private List updatedRows = new ArrayList(); private final Stopwatch stopwatch; private final Stopwatch stopwatch1; private static String modelPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz"; @@ -99,7 +98,6 @@ public class MYSQLDatahandler { this.stopwatch = Stopwatch.createUnstarted(); this.stopwatch1 = Stopwatch.createStarted(); this.stringCache = new MapMaker().concurrencyLevel(2).makeMap(); - this.multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); } public static void shiftReduceParserInitiate() { @@ -186,8 +184,8 @@ public class MYSQLDatahandler { public synchronized void checkIfUpdateMatrixes() { refreshMatrixFromDB = false; - int calculationBoundaries = 5; - int updateBadgesInteger = 250; + int updateBadgesInteger = 900; + int counter = 0; if (stopwatch1.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS1) { refreshMatrixFromDB = true; lHMSMX = DataMapper.getAllRelationScores(); @@ -195,162 +193,90 @@ public class MYSQLDatahandler { } //requiring atleast 10 entries ensures no issues in case of empty stringcache if (stringCache.values().size() > 10 && !refreshMatrixFromDB) { - if (counter <= 5) { + ConcurrentMap stringCachelocal = stringCache; + int selectUpdate = -1; + LinkedHashMap> LHMSMXLocal = lHMSMX; + int ij2 = 0; + for (String str : stringCachelocal.values()) { + if (!LHMSMXLocal.keySet().contains(str)) { + selectUpdate = ij2; + LinkedHashMap newLHM = new LinkedHashMap(); + lHMSMX.put(str, newLHM); + break; + } + ij2++; + } + if (selectUpdate == -1) { + ij2 = 0; + int valueSize = stringCachelocal.size(); + for (LinkedHashMap values : LHMSMXLocal.values()) { + int lhmSize = values.size(); + if (lhmSize < valueSize) { + selectUpdate = ij2; + } + ij2++; + } + } + semeticsUpdateCount = selectUpdate; + int beginindex = semeticsUpdateCount; + System.out.println("beginindex: " + beginindex + "\n"); + ConcurrentMap strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap(); + String get = stringCachelocal.get(beginindex); + strIndexNavigator.put(strIndexNavigator.size() + 1, get); + ConcurrentMap strIndexAll = stringCachelocal; + ConcurrentMap randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap(); + int indexes = updateBadgesInteger; + if (indexes >= strIndexAll.size()) { + indexes = strIndexAll.size() - 1; + } + int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes); + int ij1 = 0; + while (beginindexes + ij1 < beginindexes + indexes) { + String get1 = strIndexAll.get(beginindexes + ij1); + randomIndexesToUpdate.put(ij1, get1); + ij1++; + } + ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); + strIndexNavigator.values().forEach((str) -> { + randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { + boolean present = false; + if ((LHMSMXLocal.containsKey(str)) && LHMSMXLocal.get(str).containsKey(str1) + || (LHMSMXLocal.containsKey(str1)) && LHMSMXLocal.get(str).containsKey(str)) { + present = true; + } + if (!present) { + SimilarityMatrix SMX = new SimilarityMatrix(str, str1); + Callable worker = new SentimentAnalyzerTest(str, str1, SMX); + futures.put(futures.size() + 1, executor.submit(worker)); + } + }); + }); + System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); + for (Future future : futures.values()) { + System.out.println("counter: " + counter + "\n"); counter++; - ConcurrentMap stringCachelocal = stringCache; - List updateLocal = updatedRows; - int random = -1; - if (!updateLocal.contains(random)) { - updatedRows.add(random); - } - Collections.sort(updateLocal); - while (updateLocal.contains(random)) { - random = new Random().nextInt(stringCachelocal.values().size() - 6); - int indexPrev = Collections.binarySearch(updateLocal, random); - int indexNext = Collections.binarySearch(updateLocal, random + 6); - //-1 will always be index 0 - if (indexPrev > 0 && indexNext > 0) { - indexPrev = updateLocal.get(indexPrev); - indexNext = updateLocal.get(indexNext); - } - random = indexPrev < random - 5 && indexNext < random ? random : -1; - } - updatedRows.add(random); - semeticsUpdateCount = random; - int beginindex = semeticsUpdateCount; - semeticsUpdateCount += calculationBoundaries; - int temp = semeticsUpdateCount; - System.out.println("beginindex: " + beginindex + "\ntemp: " + temp + "\n"); - ConcurrentMap strIndexNavigator = new MapMaker().concurrencyLevel(2).makeMap(); - int ij = 0; - while (beginindex + ij < temp) { - String get = stringCachelocal.get(beginindex + ij); - strIndexNavigator.put(ij, get); - multiprocessCalculations.put(multiprocessCalculations.size() + 1, get); - ij++; - } - LinkedHashMap> LHMSMXLocal = lHMSMX; - ConcurrentMap strIndexAll = stringCachelocal; - ConcurrentMap strIndexNavigatorL = strIndexNavigator; - ConcurrentMap randomIndexesToUpdate = new MapMaker().concurrencyLevel(2).makeMap(); - int indexes = updateBadgesInteger; - if (indexes >= strIndexAll.size()) { - indexes = strIndexAll.size() - 1; - } - int beginindexes = new Random().nextInt((strIndexAll.size()) - indexes); - int ij1 = 0; - while (beginindexes + ij1 < beginindexes + indexes) { - String get1 = strIndexAll.get(beginindexes + ij1); - randomIndexesToUpdate.put(ij1, get1); - ij1++; - } - ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - strIndexNavigatorL.values().forEach((str) -> { - randomIndexesToUpdate.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { - boolean present = false; - if (multiprocessCalculations.values().contains(str1)) { - present = true; - } else if (LHMSMXLocal.containsKey(str)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str); - if (orDefault.containsKey(str1)) { - present = true; - } - } else if (LHMSMXLocal.containsKey(str1)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str1); - if (orDefault.containsKey(str)) { - present = true; - } - } - if (!present) { - SimilarityMatrix SMX = new SimilarityMatrix(str, str1); - Callable worker = new SentimentAnalyzerTest(str, str1, SMX); - futures.put(futures.size() + 1, executor.submit(worker)); - } - }); - }); try { - System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); - for (Future future : futures.values()) { - SimilarityMatrix SMX = future.get(); - LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); - if (get == null) { - get = new LinkedHashMap(); - } - get.put(SMX.getSecondaryString(), SMX.getDistance()); - lHMSMX.put(SMX.getPrimaryString(), get); - matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); + SimilarityMatrix SMX = future.get(20, TimeUnit.SECONDS); + LinkedHashMap getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); + if (getFuture == null) { + getFuture = new LinkedHashMap(); } - } catch (InterruptedException | ExecutionException ex) { + getFuture.put(SMX.getSecondaryString(), SMX.getDistance()); + lHMSMX.put(SMX.getPrimaryString(), getFuture); + matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); + } catch (InterruptedException | ExecutionException | TimeoutException ex) { Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); } - new Thread(() -> { - try { - if (!matrixUpdateList.isEmpty()) { - DataMapper.insertSementicMatrixes(matrixUpdateList); - System.out.println("finished datamapper semetic insert"); - } - } catch (CustomError ex) { - Logger.getLogger(MYSQLDatahandler.class - .getName()).log(Level.SEVERE, null, ex); - } - }).start(); - } else { - ConcurrentMap stringCachelocal = stringCache; - ConcurrentMap matrixUpdateList = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - Collection values = multiprocessCalculations.values(); - LinkedHashMap> LHMSMXLocal = lHMSMX; - values.forEach((str) -> { - stringCachelocal.values().stream().filter((str1) -> (!str.equals(str1))).forEachOrdered((str1) -> { - boolean present = false; - if (LHMSMXLocal.containsKey(str)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str); - if (orDefault.containsKey(str1)) { - present = true; - } - } else if (LHMSMXLocal.containsKey(str1)) { - LinkedHashMap orDefault = LHMSMXLocal.get(str1); - if (orDefault.containsKey(str)) { - present = true; - } - } - if (!present) { - SimilarityMatrix SMX = new SimilarityMatrix(str, str1); - Callable worker = new SentimentAnalyzerTest(str, str1, SMX); - futures.put(futures.size() + 1, executor.submit(worker)); - } - }); - }); - try { - System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); - for (Future future : futures.values()) { - SimilarityMatrix SMX = future.get(); - LinkedHashMap get = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); - if (get == null) { - get = new LinkedHashMap(); - } - get.put(SMX.getSecondaryString(), SMX.getDistance()); - lHMSMX.put(SMX.getPrimaryString(), get); - matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); - } - } catch (InterruptedException | ExecutionException ex) { - Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + } + try { + if (!matrixUpdateList.isEmpty()) { + DataMapper.insertSementicMatrixes(matrixUpdateList); + System.out.println("finished datamapper semetic insert"); } - multiprocessCalculations = new MapMaker().concurrencyLevel(2).makeMap(); - updatedRows = new ArrayList(); - counter = 0; - new Thread(() -> { - try { - if (!matrixUpdateList.isEmpty()) { - DataMapper.insertSementicMatrixes(matrixUpdateList); - System.out.println("finished datamapper semetic insert"); - } - } catch (CustomError ex) { - Logger.getLogger(MYSQLDatahandler.class - .getName()).log(Level.SEVERE, null, ex); - } - }).start(); + } catch (CustomError ex) { + Logger.getLogger(MYSQLDatahandler.class + .getName()).log(Level.SEVERE, null, ex); } } } @@ -439,7 +365,7 @@ public class MYSQLDatahandler { int index = 0; for (Future future : futureslocal.values()) { try { - SimilarityMatrix SMX = future.get(); + SimilarityMatrix SMX = future.get((long) 20, TimeUnit.SECONDS); double distance = SMX.getDistance(); System.out.println("index: " + index + "\nfutures size: " + futureslocal.values().size() + "\nScore: " + SMX.getDistance() + "\nSecondary: " + SMX.getSecondaryString() + "\nPrimary: " + SMX.getPrimaryString() + "\n"); @@ -447,7 +373,7 @@ public class MYSQLDatahandler { Score = distance; SMXreturn = SMX; } - } catch (InterruptedException | ExecutionException ex) { + } catch (InterruptedException | ExecutionException | TimeoutException ex) { System.out.println("ex: " + ex.getMessage() + "\n"); } index++; @@ -464,26 +390,18 @@ public class MYSQLDatahandler { concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> { futures.put(futures.size() + 1, executor.submit(worker)); }); - try { - for (Future future : futures.values()) { + for (Future future : futures.values()) { + try { DistanceObject d = future.get(); - try { - if (d.getSentence() != null && d.getDistance() != null) { - int distance = d.getDistance(); - System.out.println("distance: " + distance + "\n"); - if (distance < minDistance) { - minDistance = distance; - similar = d.getSentence(); - } - } - } catch (NullPointerException ex) { - System.out.println("failed future\n"); - + int distance = d.getDistance(); + System.out.println("distance: " + distance + "\n"); + if (distance < minDistance) { + minDistance = distance; + similar = d.getSentence(); } + } catch (NullPointerException | InterruptedException | ExecutionException ex) { + System.out.println("failed future\n"); } - } catch (InterruptedException | ExecutionException ex) { - Logger.getLogger(MYSQLDatahandler.class - .getName()).log(Level.SEVERE, null, ex); } return similar; } diff --git a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java index df7bcd20..b0981850 100644 --- a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java +++ b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java @@ -45,7 +45,7 @@ public class DiscordHandler { }).start(); MYSQLDatahandler.shiftReduceParserInitiate(); MYSQLDatahandler.instance.instantiateExecutor(); - while (MYSQLDatahandler.instance.getlHMSMXSize() < (MYSQLDatahandler.instance.getstringCacheSize() + while (MYSQLDatahandler.instance.getlHMSMXSize() / 2 < (MYSQLDatahandler.instance.getstringCacheSize() * MYSQLDatahandler.instance.getstringCacheSize()) - MYSQLDatahandler.instance.getstringCacheSize()) { MYSQLDatahandler.instance.checkIfUpdateMatrixes(); }