diff --git a/ArtificialAutism/src/main/java/DataLayer/DBCPDataSourceHLstats.java b/ArtificialAutism/src/main/java/DataLayer/DBCPDataSourceHLstats.java new file mode 100644 index 00000000..ba9a61f1 --- /dev/null +++ b/ArtificialAutism/src/main/java/DataLayer/DBCPDataSourceHLstats.java @@ -0,0 +1,42 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package DataLayer; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.commons.dbcp2.BasicDataSource; + +/** + * + * @author install1 + */ +public class DBCPDataSourceHLstats { + + private static BasicDataSource ds = new BasicDataSource(); + static { + try { + ds.setDriver(new com.mysql.cj.jdbc.Driver()); + ds.setUrl("jdbc:mysql://151.80.230.149:3306/unloze_stats?useLegacyDatetimeCode=false&serverTimezone=UTC"); + ds.setUsername("unloze_stats"); + ds.setPassword("R8J8E9Fzmcc7ZfDAGAk7"); + ds.setMaxTotal(-1); + ds.setMinIdle(5); + ds.setMaxIdle(-1); + ds.setMaxOpenPreparedStatements(100); + } catch (SQLException ex) { + Logger.getLogger(DBCPDataSource.class.getName()).log(Level.SEVERE, null, ex); + } + } + + public static Connection getConnection() throws SQLException { + return ds.getConnection(); + } + + private DBCPDataSourceHLstats() { + } +} diff --git a/ArtificialAutism/src/main/java/DataLayer/DataMapper.java b/ArtificialAutism/src/main/java/DataLayer/DataMapper.java index bab5eab6..824e4e92 100644 --- a/ArtificialAutism/src/main/java/DataLayer/DataMapper.java +++ b/ArtificialAutism/src/main/java/DataLayer/DataMapper.java @@ -179,6 +179,24 @@ public class DataMapper { return LHMSMX; } + public static ConcurrentMap getHLstatsMessages() { + ConcurrentMap hlStatsMessages = new MapMaker().concurrencyLevel(2).makeMap(); + try (Connection l_cCon = DBCPDataSourceHLstats.getConnection()) { + String l_sSQL = "SELECT message FROM `hlstats_Events_Chat`"; + try (PreparedStatement l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY, + java.sql.ResultSet.CONCUR_READ_ONLY)) { + try (ResultSet l_rsSearch = l_pStatement.executeQuery()) { + while (l_rsSearch.next()) { + hlStatsMessages.put(hlStatsMessages.size() + 1, l_rsSearch.getString(1)); + } + } + } + } catch (SQLException ex) { + Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex); + } + return hlStatsMessages; + } + public static void CloseConnections(PreparedStatement ps, ResultSet rs, Connection con) { if (rs != null) { try { @@ -202,5 +220,4 @@ public class DataMapper { } } } - } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/LevenshteinDistance.java b/ArtificialAutism/src/main/java/FunctionLayer/LevenshteinDistance.java index 4134e710..b4351d58 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/LevenshteinDistance.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/LevenshteinDistance.java @@ -5,28 +5,23 @@ */ package FunctionLayer; +import java.util.Map; import java.util.concurrent.Callable; /** * * @author install1 */ -public class LevenshteinDistance implements Callable { +public class LevenshteinDistance implements Callable> { private CharSequence lhs; private CharSequence rhs; - private DistanceObject dco; + private Map.Entry distanceEntry; private static int minimum(int a, int b, int c) { return Math.min(Math.min(a, b), c); } - public LevenshteinDistance(CharSequence lhs, CharSequence rhs, DistanceObject dco) { - this.lhs = lhs; - this.rhs = rhs; - this.dco = dco; - } - public LevenshteinDistance(CharSequence lhs, CharSequence rhs) { this.lhs = lhs; this.rhs = rhs; @@ -52,7 +47,7 @@ public class LevenshteinDistance implements Callable { } @Override - public DistanceObject call() { + public Map.Entry call() { try { int[][] distance = new int[lhs.length() + 1][rhs.length() + 1]; @@ -70,12 +65,11 @@ public class LevenshteinDistance implements Callable { distance[i - 1][j - 1] + ((lhs.charAt(i - 1) == rhs.charAt(j - 1)) ? 0 : 1)); } } - dco.setDistance(distance[lhs.length()][rhs.length()]); + distanceEntry.setValue(distance[lhs.length()][rhs.length()]); } catch (Exception ex) { System.out.println("ex msg: " + ex.getMessage() + "\n"); - dco.setDistance(100); - return dco; + return null; } - return dco; + return distanceEntry; } } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java index 0651c685..e2f0253f 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/MYSQLDatahandler.java @@ -17,6 +17,7 @@ import edu.stanford.nlp.ling.TaggedWord; import edu.stanford.nlp.ling.Word; import edu.stanford.nlp.parser.lexparser.LexicalizedParser; import edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser; +import edu.stanford.nlp.pipeline.Annotation; import edu.stanford.nlp.pipeline.StanfordCoreNLP; import edu.stanford.nlp.process.DocumentPreprocessor; import edu.stanford.nlp.tagger.maxent.MaxentTagger; @@ -33,11 +34,9 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Random; -import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -57,17 +56,20 @@ public class MYSQLDatahandler { private volatile boolean refreshMatrixFromDB; private static int secondaryIterator = 0; private final ConcurrentMap stringCache; + private static ConcurrentMap pipelineAnnotationCache; + private static ConcurrentMap pipelineSentimentAnnotationCache; + private static ConcurrentMap jmweAnnotationCache; private LinkedHashMap> lHMSMX = new LinkedHashMap(); private final Stopwatch stopwatch; private final Stopwatch stopwatch1; - private ExecutorService executor; - + private ForkJoinPool executor; private static String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz"; private static String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz"; private static String lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"; private static String taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger"; private static String nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz"; private static String jmweIndexData = "/home/javatests/lib/mweindex_wordnet3.0_semcor1.6.data"; // ./lib/mweindex_wordnet3.0_semcor1.6.data + private static String jmweIndexDataLocalTest = "E:/java8/Projects/mweindex_wordnet3.0_semcor1.6.data"; private static MaxentTagger tagger; private static ShiftReduceParser model; private static String[] options = {"-maxLength", "100"}; @@ -83,27 +85,17 @@ public class MYSQLDatahandler { private static StanfordCoreNLP pipelineSentiment; private static StanfordCoreNLP pipelineJMWE; - public static AbstractSequenceClassifier getClassifier() { - return classifier; - } - - public static void setClassifier(AbstractSequenceClassifier classifier) { - MYSQLDatahandler.classifier = classifier; - } - - public void instantiateExecutor() { - this.executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), - ForkJoinPool.defaultForkJoinWorkerThreadFactory, - null, true); - } - public MYSQLDatahandler() { this.stopwatch = Stopwatch.createUnstarted(); this.stopwatch1 = Stopwatch.createStarted(); this.stringCache = new MapMaker().concurrencyLevel(2).makeMap(); + //cant sadly just have one annotation for every pipelines, one pipeline per annotation is required + this.jmweAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap(); + this.pipelineAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap(); + this.pipelineSentimentAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap(); } - public static void shiftReduceParserInitiate() { + public void shiftReduceParserInitiate() { try { classifier = CRFClassifier.getClassifierNoExceptions(nerModel); } catch (ClassCastException ex) { @@ -127,7 +119,7 @@ public class MYSQLDatahandler { propsJMWE.setProperty("customAnnotatorClass.jmwe", "edu.stanford.nlp.pipeline.JMWEAnnotator"); propsJMWE.setProperty("customAnnotatorClass.jmwe.verbose", "false"); propsJMWE.setProperty("customAnnotatorClass.jmwe.underscoreReplacement", "-"); - propsJMWE.setProperty("customAnnotatorClass.jmwe.indexData", jmweIndexData); + propsJMWE.setProperty("customAnnotatorClass.jmwe.indexData", jmweIndexData); //jmweIndexDataLocalTest jmweIndexData propsJMWE.setProperty("customAnnotatorClass.jmwe.detector", "Exhaustive"); //"Consecutive", "Exhaustive", "ProperNouns", "Complex" and "CompositeConsecutiveProperNouns" propsJMWE.setProperty("annotators", "tokenize, ssplit, pos, lemma, jmwe"); @@ -137,8 +129,26 @@ public class MYSQLDatahandler { pipelineJMWE = new StanfordCoreNLP(propsJMWE); } - public static StanfordCoreNLP getPipelineJMWE() { - return pipelineJMWE; + public static AbstractSequenceClassifier getClassifier() { + return classifier; + } + + public static void setClassifier(AbstractSequenceClassifier classifier) { + MYSQLDatahandler.classifier = classifier; + } + + public void updateStringCache() { + try { + checkIfUpdateStrings(); + } catch (CustomError ex) { + Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + } + } + + public void instantiateExecutor() { + this.executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), + ForkJoinPool.defaultForkJoinWorkerThreadFactory, + null, false); //true } public static GrammaticalStructureFactory getGsf() { @@ -177,6 +187,7 @@ public class MYSQLDatahandler { try { DataMapper.createTables(); stringCache.putAll(getCache()); + addHLstatsMessages(); lHMSMX = DataMapper.getAllRelationScores(); } catch (CustomError ex) { Logger.getLogger(MYSQLDatahandler.class @@ -184,6 +195,52 @@ public class MYSQLDatahandler { } } + private void addHLstatsMessages() { + ConcurrentMap hlStatsMessages = DataMapper.getHLstatsMessages(); + ConcurrentMap strCacheLocal = stringCache; + int hardcap = 300; + int counter = 0; + for (String str : hlStatsMessages.values()) { + if (!str.startsWith("!")) { + boolean present = false; + for (String str1 : strCacheLocal.values()) { + if (str.equals(str1)) { + present = true; + break; + } + } + if (!present) { + MessageResponseHandler.getMessage(str); + } + } + if (counter >= hardcap) { + break; + } + counter++; + } + } + + public void instantiateAnnotationMap() { + for (String str : stringCache.values()) { + System.out.println("str annotation jmwe: " + str + "\n"); + Annotation strAnno = new Annotation(str); + pipelineJMWE.annotate(strAnno); + jmweAnnotationCache.put(str, strAnno); + } + for (String str : stringCache.values()) { + System.out.println("str annotation pipeline: " + str + "\n"); + Annotation strAnno = new Annotation(str); + pipeline.annotate(strAnno); + pipelineAnnotationCache.put(str, strAnno); + } + for (String str : stringCache.values()) { + System.out.println("str annotation pipelinesentiment: " + str + "\n"); + Annotation strAnno = new Annotation(str); + pipelineSentiment.annotate(strAnno); + pipelineSentimentAnnotationCache.put(str, strAnno); + } + } + public synchronized void checkIfUpdateMatrixes() { refreshMatrixFromDB = false; int counter = 0; @@ -233,47 +290,52 @@ public class MYSQLDatahandler { boolean present = false; LinkedHashMap orDefault = lHMSMX.getOrDefault(str, null); if (orDefault != null) { - Double orDefault1 = orDefault.getOrDefault(str1, null); - if (orDefault1 != null) { - // System.out.println("present true \nstr: " + str + "\nstr1: " + str1 + "\n"); - present = true; + for (String strkey : orDefault.keySet()) { + if (strkey.equals(str1)) { + present = true; + break; + } } } if (!present) { orDefault = lHMSMX.getOrDefault(str1, null); if (orDefault != null) { - Double orDefault1 = orDefault.getOrDefault(str, null); - if (orDefault1 != null) { - // System.out.println("present true \nstr: " + str + "\nstr1: " + str1 + "\n"); - present = true; + for (String strkey : orDefault.keySet()) { + if (strkey.equals(str)) { + present = true; + break; + } } } } if (!present) { - SimilarityMatrix SMX = new SimilarityMatrix(str, str1); - Callable worker = new SentimentAnalyzerTest(str, str1, SMX); - futures.put(futures.size() + 1, executor.submit(worker)); LinkedHashMap orDefault1 = lHMSMX.getOrDefault(str, null); if (orDefault1 == null) { orDefault1 = new LinkedHashMap(); } orDefault1.put(str1, 0.0); lHMSMX.put(str, orDefault1); + SimilarityMatrix SMX = new SimilarityMatrix(str, str1); + Callable worker = new SentimentAnalyzerTest(str, str1, SMX, jmweAnnotationCache.get(str), + jmweAnnotationCache.get(str1), pipelineAnnotationCache.get(str), pipelineAnnotationCache.get(str1), + pipelineSentimentAnnotationCache.get(str), pipelineSentimentAnnotationCache.get(str1)); + futures.put(futures.size() + 1, executor.submit(worker)); } }); }); System.out.println("finished worker assignment, futures size: " + futures.size() + "\n"); for (Future future : futures.values()) { - System.out.println("counter: " + counter + "\n"); - counter++; SimilarityMatrix SMX = new SimilarityMatrix("", ""); try { SMX = future.get(20, TimeUnit.SECONDS); } catch (InterruptedException | ExecutionException | TimeoutException ex) { Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex); + SMX = null; } - LinkedHashMap getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); - if (getFuture != null) { + System.out.println("counter: " + counter + "\nSMX: " + SMX == null ? "SMX null\n" : "SMX not null\n"); + counter++; + if (SMX != null) { + LinkedHashMap getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null); getFuture.put(SMX.getSecondaryString(), SMX.getDistance()); lHMSMX.put(SMX.getPrimaryString(), getFuture); matrixUpdateList.put(matrixUpdateList.size() + 1, SMX); @@ -309,6 +371,15 @@ public class MYSQLDatahandler { for (String str1 : str.values()) { stringCache.put(j, str1); j++; + Annotation strAnno = new Annotation(str1); + pipelineJMWE.annotate(strAnno); + jmweAnnotationCache.put(str1, strAnno); + strAnno = new Annotation(str1); + pipeline.annotate(strAnno); + pipelineAnnotationCache.put(str1, strAnno); + strAnno = new Annotation(str1); + pipelineSentiment.annotate(strAnno); + pipelineSentimentAnnotationCache.put(str1, strAnno); } if (!stopwatch.isRunning()) { stopwatch.start(); @@ -329,7 +400,7 @@ public class MYSQLDatahandler { SimilarityMatrix SMXreturn = new SimilarityMatrix("", ""); System.out.println("pre mostSimilarSTR \n"); String mostSimilarSTR = mostSimilar(str, strArrs); - if (mostSimilarSTR != null && !mostSimilarSTR.isEmpty()) { + if (mostSimilarSTR != null) { System.out.println("mostSimilarSTR; " + mostSimilarSTR + "\n"); LinkedHashMap orDefault = LHMSMXLocal.getOrDefault(mostSimilarSTR, null); if (orDefault != null) { @@ -366,14 +437,24 @@ public class MYSQLDatahandler { ConcurrentMap> futureslocal = new MapMaker().concurrencyLevel(2).makeMap(); for (String str1 : strCache.values()) { if (!str.equals(str1)) { - Callable worker = new SentimentAnalyzerTest(str, str1, new SimilarityMatrix(str, str1)); + SimilarityMatrix SMX = new SimilarityMatrix(str, str1); + Callable worker = new SentimentAnalyzerTest(str, str1, SMX, + jmweAnnotationCache.get(str), jmweAnnotationCache.get(str1), pipelineAnnotationCache.get(str), + pipelineAnnotationCache.get(str1), pipelineSentimentAnnotationCache.get(str), + pipelineSentimentAnnotationCache.get(str1)); futureslocal.put(futureslocal.size() + 1, executor.submit(worker)); } } int index = 0; for (Future future : futureslocal.values()) { + SimilarityMatrix SMX = new SimilarityMatrix("", ""); try { - SimilarityMatrix SMX = future.get((long) 20, TimeUnit.SECONDS); + SMX = future.get(20, TimeUnit.SECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException ex) { + System.out.println("ex getResponsemsg: " + ex.getMessage() + "\n"); + SMX = null; + } + if (SMX != null) { double distance = SMX.getDistance(); System.out.println("index: " + index + "\nfutures size: " + futureslocal.values().size() + "\nScore: " + SMX.getDistance() + "\nSecondary: " + SMX.getSecondaryString() + "\nPrimary: " + SMX.getPrimaryString() + "\n"); @@ -381,8 +462,6 @@ public class MYSQLDatahandler { Score = distance; SMXreturn = SMX; } - } catch (InterruptedException | ExecutionException | TimeoutException ex) { - System.out.println("ex: " + ex.getMessage() + "\n"); } index++; } @@ -394,18 +473,19 @@ public class MYSQLDatahandler { public String mostSimilar(String toBeCompared, ConcurrentMap concurrentStrings) { int minDistance = 8; String similar = ""; - ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> { + ConcurrentMap>> futures = new MapMaker().concurrencyLevel(2).makeMap(); + for (String str : concurrentStrings.values()) { + Callable> worker = new LevenshteinDistance(toBeCompared, str); futures.put(futures.size() + 1, executor.submit(worker)); - }); - for (Future future : futures.values()) { + } + for (Future> future : futures.values()) { try { - DistanceObject d = future.get(); - int distance = d.getDistance(); - System.out.println("distance: " + distance + "\n"); + Entry futureEntry = future.get(); + int distance = futureEntry.getValue(); if (distance < minDistance) { + System.out.println("distance: " + distance + "\n"); minDistance = distance; - similar = d.getSentence(); + similar = futureEntry.getKey(); } } catch (NullPointerException | InterruptedException | ExecutionException ex) { System.out.println("failed future\n"); @@ -605,17 +685,10 @@ public class MYSQLDatahandler { ConcurrentMap returnmap = new MapMaker().concurrencyLevel(2).makeMap(); ConcurrentMap allStrings = stringCache; int intervalAprove = allStrings.values().size() / 10; - String str1 = "and to revise the questions and materials and such"; - String str2 = "as William said earlier. Visiting your dentist once or twice a year is enough"; - String str3 = "At least, you have more time to prepare then"; for (String str : strmap.values()) { + int counter = 0; + boolean calculationIssues = false; ConcurrentMap> futures = new MapMaker().concurrencyLevel(2).makeMap(); - Callable worker = new SentimentAnalyzerTest(str, str1, new SimilarityMatrix(str, str1)); - futures.put(futures.size() + 1, executor.submit(worker)); - worker = new SentimentAnalyzerTest(str, str2, new SimilarityMatrix(str, str2)); - futures.put(futures.size() + 1, executor.submit(worker)); - worker = new SentimentAnalyzerTest(str, str3, new SimilarityMatrix(str, str3)); - futures.put(futures.size() + 1, executor.submit(worker)); int ij2 = 0; if (allStrings.isEmpty()) { allStrings = strmap; @@ -624,15 +697,24 @@ public class MYSQLDatahandler { if (ij2 > intervalAprove) { break; } - worker = new SentimentAnalyzerTest(str, strValues, new SimilarityMatrix(str, strValues)); + Callable worker = new SentimentAnalyzerTest(str, strValues, new SimilarityMatrix(str, strValues), + jmweAnnotationCache.get(str), jmweAnnotationCache.get(strValues), pipelineAnnotationCache.get(str), + pipelineAnnotationCache.get(strValues), pipelineSentimentAnnotationCache.get(str), + pipelineSentimentAnnotationCache.get(strValues)); futures.put(futures.size() + 1, executor.submit(worker)); ij2++; } - int counter = 0; - boolean calculationIssues = false; for (Future future : futures.values()) { try { - future.get(20, TimeUnit.SECONDS); + SimilarityMatrix get = future.get(20, TimeUnit.SECONDS); + if (get == null) { + System.out.println("counter timeouts: " + counter + "\n"); + counter++; + if (counter >= 10) { + calculationIssues = true; + break; + } + } } catch (InterruptedException | ExecutionException | TimeoutException ex) { System.out.println("counter timeouts: " + counter + "\n"); counter++; diff --git a/ArtificialAutism/src/main/java/FunctionLayer/SimilarityMatrix.java b/ArtificialAutism/src/main/java/FunctionLayer/SimilarityMatrix.java index 9991b162..1445a0e4 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/SimilarityMatrix.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/SimilarityMatrix.java @@ -41,49 +41,6 @@ public class SimilarityMatrix{ this.distance = result; } - /* - public double getDistanceCalculations() { - ILexicalDatabase db = new NictWordNet(); - WS4JConfiguration.getInstance().setMFS(true); - RelatednessCalculator rc1 = new WuPalmer(db); - RelatednessCalculator rc2 = new Resnik(db); - RelatednessCalculator rc3 = new JiangConrath(db); - RelatednessCalculator rc4 = new Lin(db); - RelatednessCalculator rc5 = new LeacockChodorow(db); - RelatednessCalculator rc6 = new Path(db); - RelatednessCalculator rc7 = new Lesk(db); - RelatednessCalculator rc8 = new HirstStOnge(db); - double maxScore = -1D; - List RCList = new ArrayList(); - RCList.add(rc1); - RCList.add(rc2); - RCList.add(rc3); - RCList.add(rc4); - RCList.add(rc5); - RCList.add(rc6); - RCList.add(rc7); - RCList.add(rc8); - for (RelatednessCalculator rc : RCList) { - double s = rc.calcRelatednessOfWords(PrimaryString, SecondaryString); - s /= 1000; - if (s > 0.000000) { - System.out.println("s: " + s + "\n" + " PrimaryString: " + PrimaryString + "\n" + " SecondaryString: " + SecondaryString + "\n" - + " rc: " + rc.toString() + "\n"); - } - String str = String.format("%.12f", s); - if (str.contains(",")) { - str = str.substring(0, str.indexOf(",")); - } - int strend = str.length() > 6 ? 6 : str.length(); - str = str.substring(0, strend); - double score = Double.valueOf(str); - if (score > maxScore) { - maxScore = score; - } - } - return maxScore == -1D ? 0.00 : maxScore; - } - */ public String getPrimaryString() { return PrimaryString; } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java index 78ae931d..6cde4faa 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/StanfordParser/SentimentAnalyzerTest.java @@ -53,7 +53,6 @@ import org.ejml.simple.SimpleMatrix; public class SentimentAnalyzerTest implements Callable { private SimilarityMatrix smxParam; - private String str; private String str1; private ShiftReduceParser model; @@ -61,372 +60,364 @@ public class SentimentAnalyzerTest implements Callable { private GrammaticalStructureFactory gsf; private StanfordCoreNLP pipeline; private StanfordCoreNLP pipelineSentiment; - private StanfordCoreNLP pipelineJMWE; private AbstractSequenceClassifier classifier; + private Annotation jmweStrAnnotation1; + private Annotation jmweStrAnnotation2; + private Annotation pipelineAnnotation1; + private Annotation pipelineAnnotation2; + private Annotation pipelineAnnotation1Sentiment; + private Annotation pipelineAnnotation2Sentiment; - public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam) { + public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam, Annotation str1Annotation, Annotation str2Annotation, + Annotation strPipeline1, Annotation strPipeline2, Annotation strPipeSentiment1, Annotation strPipeSentiment2) { this.str = str; this.str1 = str1; this.smxParam = smxParam; - model = MYSQLDatahandler.getModel(); - tagger = MYSQLDatahandler.getTagger(); - pipeline = MYSQLDatahandler.getPipeline(); - pipelineSentiment = MYSQLDatahandler.getPipelineSentiment(); - pipelineJMWE = MYSQLDatahandler.getPipelineJMWE(); - gsf = MYSQLDatahandler.getGsf(); - classifier = MYSQLDatahandler.getClassifier(); + this.model = MYSQLDatahandler.getModel(); + this.tagger = MYSQLDatahandler.getTagger(); + this.pipeline = MYSQLDatahandler.getPipeline(); + this.pipelineSentiment = MYSQLDatahandler.getPipelineSentiment(); + this.gsf = MYSQLDatahandler.getGsf(); + this.classifier = MYSQLDatahandler.getClassifier(); + this.jmweStrAnnotation1 = str1Annotation; + this.jmweStrAnnotation2 = str2Annotation; + this.pipelineAnnotation1 = strPipeline1; + this.pipelineAnnotation2 = strPipeline2; + this.pipelineAnnotation1Sentiment = strPipeSentiment1; //maybe process? + this.pipelineAnnotation2Sentiment = strPipeSentiment2; } @Override public SimilarityMatrix call() { - try { - Double score = -100.0; - List> taggedwordlist1 = new ArrayList(); - List> taggedwordlist2 = new ArrayList(); - DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1)); - for (List sentence : tokenizer) { - taggedwordlist1.add(model.apply(tagger.tagSentence(sentence)).taggedYield()); - } - tokenizer = new DocumentPreprocessor(new StringReader(str)); - for (List sentence : tokenizer) { - taggedwordlist2.add(model.apply(tagger.tagSentence(sentence)).taggedYield()); - } - int counter = 0; - int counter1 = 0; - counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum); - counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum); - int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter; - overValue *= 16; - score -= overValue; - ConcurrentMap tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap(); - taggedwordlist1.forEach((TGWList) -> { - TGWList.forEach((TaggedWord) -> { - if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) { - tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag()); - } - }); - }); - taggedwordlist1.clear(); - AtomicInteger runCount = new AtomicInteger(0); - taggedwordlist2.forEach((TGWList) -> { - TGWList.forEach((TaggedWord) -> { - if (tgwlistIndex.values().contains(TaggedWord.tag())) { - tgwlistIndex.values().remove(TaggedWord.tag()); - runCount.getAndIncrement(); - } - }); - }); - tgwlistIndex.clear(); - taggedwordlist2.clear(); - score += runCount.get() * 64; - Annotation annotation = new Annotation(str1); - pipeline.annotate(annotation); - ConcurrentMap sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap(); - for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) { - Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); - sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse); - } - Annotation annotation1 = new Annotation(str); - pipeline.annotate(annotation1); - for (CoreMap sentence : annotation1.get(CoreAnnotations.SentencesAnnotation.class)) { - Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); - GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse); - Collection allTypedDependencies = gs.allTypedDependencies(); - ConcurrentMap filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap(); - for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) { - Set inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1); - Set inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse); - ConcurrentMap constiLabels = new MapMaker().concurrencyLevel(2).makeMap(); - for (Constituent consti : inT1notT2) { - for (Constituent consti1 : inT2notT1) { - if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) { - score += 64; - constiLabels.put(constiLabels.size(), consti.value()); - } - } - } - GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1); - Collection allTypedDependencies1 = gs1.allTypedDependencies(); - for (TypedDependency TDY1 : allTypedDependencies1) { - IndexedWord dep = TDY1.dep(); - IndexedWord gov = TDY1.gov(); - GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep); - if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) { - score += 900; - } - GrammaticalRelation reln = TDY1.reln(); - if (reln.isApplicable(sentenceConstituencyParse)) { - score += 256; - } - } - for (TypedDependency TDY : allTypedDependencies) { - IndexedWord dep = TDY.dep(); - IndexedWord gov = TDY.gov(); - GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep); - if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) { - score += 900; - } - GrammaticalRelation reln = TDY.reln(); - if (reln.isApplicable(sentenceConstituencyParse1)) { - score += 256; - } - } - AtomicInteger runCount1 = new AtomicInteger(0); - sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> { - sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma()) - && !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> { - filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma()); - return _item; - }).forEachOrdered((_item) -> { - runCount1.getAndIncrement(); - }); - }); - score += runCount1.get() * 1500; + Double score = -100.0; + List> taggedwordlist1 = new ArrayList(); + List> taggedwordlist2 = new ArrayList(); + DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1)); + for (List sentence : tokenizer) { + taggedwordlist1.add(model.apply(tagger.tagSentence(sentence)).taggedYield()); + } + tokenizer = new DocumentPreprocessor(new StringReader(str)); + for (List sentence : tokenizer) { + taggedwordlist2.add(model.apply(tagger.tagSentence(sentence)).taggedYield()); + } + int counter = 0; + int counter1 = 0; + counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum); + counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum); + int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter; + overValue *= 16; + score -= overValue; + ConcurrentMap tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap(); + taggedwordlist1.forEach((TGWList) -> { + TGWList.forEach((TaggedWord) -> { + if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) { + tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag()); } - } - sentenceConstituencyParseList.clear(); - Annotation annotationSentiment1 = pipelineSentiment.process(str); - ConcurrentMap simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap sentiment1 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap sentiment2 = new MapMaker().concurrencyLevel(2).makeMap(); - for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) { - Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); - sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree)); - SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); - SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); - simpleSMXlist.put(simpleSMXlist.size(), predictions); - simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector); - } - annotationSentiment1 = pipelineSentiment.process(str1); - for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) { - Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); - sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree)); - SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); - SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); - score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> { - subtracter *= 25; - return subtracter; - }).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item); - for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) { - double dot = nodeVector.dot(simpleSMX); - double elementSum = nodeVector.kron(simpleSMX).elementSum(); - elementSum = Math.round(elementSum * 100.0) / 100.0; - if (dot < 0.1) { + }); + }); + taggedwordlist1.clear(); + AtomicInteger runCount = new AtomicInteger(0); + taggedwordlist2.forEach((TGWList) -> { + TGWList.forEach((TaggedWord) -> { + if (tgwlistIndex.values().contains(TaggedWord.tag())) { + tgwlistIndex.values().remove(TaggedWord.tag()); + runCount.getAndIncrement(); + } + }); + }); + tgwlistIndex.clear(); + taggedwordlist2.clear(); + score += runCount.get() * 64; + ConcurrentMap sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap(); + for (CoreMap sentence : pipelineAnnotation1.get(CoreAnnotations.SentencesAnnotation.class)) { + Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); + sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse); + } + for (CoreMap sentence : pipelineAnnotation2.get(CoreAnnotations.SentencesAnnotation.class)) { + Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class); + GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse); + Collection allTypedDependencies = gs.allTypedDependencies(); + ConcurrentMap filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap(); + for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) { + Set inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1); + Set inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse); + ConcurrentMap constiLabels = new MapMaker().concurrencyLevel(2).makeMap(); + for (Constituent consti : inT1notT2) { + for (Constituent consti1 : inT2notT1) { + if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) { + score += 64; + constiLabels.put(constiLabels.size(), consti.value()); + } + } + } + GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1); + Collection allTypedDependencies1 = gs1.allTypedDependencies(); + for (TypedDependency TDY1 : allTypedDependencies1) { + IndexedWord dep = TDY1.dep(); + IndexedWord gov = TDY1.gov(); + GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep); + if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) { + score += 900; + } + GrammaticalRelation reln = TDY1.reln(); + if (reln.isApplicable(sentenceConstituencyParse)) { score += 256; } - if (elementSum < 0.1 && elementSum > 0.0) { - score += 1300; - } else if (elementSum > 0.1 && elementSum < 1.0) { - score -= 1100; - } else { - score -= 1424; + } + for (TypedDependency TDY : allTypedDependencies) { + IndexedWord dep = TDY.dep(); + IndexedWord gov = TDY.gov(); + GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep); + if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) { + score += 900; + } + GrammaticalRelation reln = TDY.reln(); + if (reln.isApplicable(sentenceConstituencyParse1)) { + score += 256; } } + AtomicInteger runCount1 = new AtomicInteger(0); + sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> { + sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma()) + && !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> { + filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma()); + return _item; + }).forEachOrdered((_item) -> { + runCount1.getAndIncrement(); + }); + }); + score += runCount1.get() * 1500; } - score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500; - DocumentReaderAndWriter readerAndWriter = classifier.makePlainTextReaderAndWriter(); - List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter); - List classifyRaw2 = classifier.classifyRaw(str1, readerAndWriter); - score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200; - Annotation annotationSentiment = pipelineSentiment.process(str); - int mainSentiment1 = 0; - int longest1 = 0; - int mainSentiment2 = 0; - int longest2 = 0; - for (CoreMap sentence : annotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) { - Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); - int sentiment = RNNCoreAnnotations.getPredictedClass(tree); - String partText = sentence.toString(); - SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); - if (partText.length() > longest1) { - mainSentiment1 = sentiment; - longest1 = partText.length(); - } - } - annotationSentiment = pipelineSentiment.process(str1); - for (CoreMap sentence : annotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) { - Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); - int sentiment = RNNCoreAnnotations.getPredictedClass(tree); - SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); - String partText = sentence.toString(); - if (partText.length() > longest2) { - mainSentiment2 = sentiment; - longest2 = partText.length(); - } - } - if (longest1 != longest2) { - long deffLongest = longest1 > longest2 ? longest1 : longest2; - long deffshorter = longest1 < longest2 ? longest1 : longest2; - if (deffLongest >= (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) { - score += (deffLongest - deffshorter) * 200; - } else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) { - score += (deffLongest - deffshorter) * 200; - } else { - score -= (deffLongest - deffshorter) * 50; - } - } - Annotation jmweStrAnnotation = new Annotation(str); - pipelineJMWE.annotate(jmweStrAnnotation); - List sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class); - int tokensCounter1 = 0; - int tokensCounter2 = 0; - int anotatorcounter1 = 0; - int anotatorcounter2 = 0; - int inflectedCounterPositive1 = 0; - int inflectedCounterPositive2 = 0; - int inflectedCounterNegative = 0; - int MarkedContinuousCounter1 = 0; - int MarkedContinuousCounter2 = 0; - int UnmarkedPatternCounter = 0; - ConcurrentMap ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap strTokenStems2 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap strTokenForm1 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap strTokenForm2 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap strTokenGetEntry1 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap strTokenGetEntry2 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap strTokenGetiPart1 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap strTokenGetiPart2 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap strTokenEntryPOS1 = new MapMaker().concurrencyLevel(2).makeMap(); - ConcurrentMap strTokenEntryPOS2 = new MapMaker().concurrencyLevel(2).makeMap(); - for (CoreMap sentence : sentences) { - for (IMWE token : sentence.get(JMWEAnnotation.class)) { - if (token.isInflected()) { - inflectedCounterPositive1++; - } else { - inflectedCounterNegative++; - } - strTokenForm1.put(strTokenForm1.size() + 1, token.getForm()); - strTokenGetEntry1.put(strTokenGetEntry1.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1)); - Collection values = token.getPartMap().values(); - IMWEDesc entry = token.getEntry(); - MarkedContinuousCounter1 += entry.getMarkedContinuous(); - UnmarkedPatternCounter += entry.getUnmarkedPattern(); - for (IMWEDesc.IPart iPart : values) { - strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm()); - } - for (String strPostPrefix : entry.getPOS().getPrefixes()) { - strTokenEntryPOS1.put(strTokenEntryPOS1.size() + 1, strPostPrefix); - } - for (IToken tokens : token.getTokens()) { - ITokenMapTag1.put(ITokenMapTag1.size() + 1, tokens.getTag()); - for (String strtoken : tokens.getStems()) { - strTokenStems1.put(strTokenStems1.size() + 1, strtoken); - } - } - tokensCounter1++; - } - anotatorcounter1++; - } - jmweStrAnnotation = new Annotation(str1); - pipelineJMWE.annotate(jmweStrAnnotation); - sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class); - for (CoreMap sentence : sentences) { - for (IMWE token : sentence.get(JMWEAnnotation.class)) { - if (token.isInflected()) { - inflectedCounterPositive2++; - } else { - inflectedCounterNegative--; - } - strTokenForm2.put(strTokenForm2.size() + 1, token.getForm()); - strTokenGetEntry2.put(strTokenGetEntry2.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1)); - Collection values = token.getPartMap().values(); - IMWEDesc entry = token.getEntry(); - MarkedContinuousCounter2 += entry.getMarkedContinuous(); - UnmarkedPatternCounter += entry.getUnmarkedPattern(); - for (IMWEDesc.IPart iPart : values) { - strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm()); - } - for (String strPostPrefix : entry.getPOS().getPrefixes()) { - strTokenEntryPOS2.put(strTokenEntryPOS2.size() + 1, strPostPrefix); - } - for (IToken tokens : token.getTokens()) { - ITokenMapTag2.put(ITokenMapTag2.size() + 1, tokens.getTag()); - for (String strtoken : tokens.getStems()) { - strTokenStems2.put(strTokenStems2.size() + 1, strtoken); - } - } - tokensCounter2++; - } - anotatorcounter2++; - } - for (String strTokenPos1 : strTokenEntryPOS1.values()) { - for (String strTokenPos2 : strTokenEntryPOS2.values()) { - if (strTokenPos1.equals(strTokenPos2)) { - score += 500; - } - } - } - score += UnmarkedPatternCounter * 1600; - if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) { - score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500 - : (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500; - } - for (String strTokeniPart1 : strTokenGetiPart1.values()) { - for (String strTokeniPart2 : strTokenGetiPart2.values()) { - if (strTokeniPart1.equals(strTokeniPart2)) { - score += 400; - } - } - } - for (String strTokenEntry1 : strTokenGetEntry1.values()) { - for (String strTokenEntry2 : strTokenGetEntry2.values()) { - if (strTokenEntry1.equals(strTokenEntry2)) { - score += 2500; - } - } - } - for (String strmapTag : ITokenMapTag1.values()) { - for (String strmapTag1 : ITokenMapTag2.values()) { - if (strmapTag.equals(strmapTag1)) { - score += 1450; - } - } - } - for (String strTokenForm1itr1 : strTokenForm1.values()) { - for (String strTokenForm1itr2 : strTokenForm2.values()) { - if (strTokenForm1itr1.equals(strTokenForm1itr2)) { - score += 2600; - } else if (strTokenForm1itr1.contains(strTokenForm1itr2)) { - score += 500; - } - } - } - for (String strTokenStem : strTokenStems1.values()) { - for (String strTokenStem1 : strTokenStems2.values()) { - if (strTokenStem.equals(strTokenStem1)) { - score += 1500; - } - } - } - if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) { - score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650; - } - if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) { - score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550; - } - if (anotatorcounter1 > 1 && anotatorcounter2 > 1) { - score += (anotatorcounter1 + anotatorcounter2) * 400; - } - if (tokensCounter1 > 0 && tokensCounter2 > 0) { - score += (tokensCounter1 + tokensCounter2) * 400; - } else { - score -= tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500; - } - LevenshteinDistance leven = new LevenshteinDistance(str, str1); - int SentenceScoreDiff = leven.computeLevenshteinDistance(); - SentenceScoreDiff *= 15; - score -= SentenceScoreDiff; - System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n"); - smxParam.setDistance(score); - } catch (Exception ex) { - System.out.println("ex: " + ex.getMessage() + "\n"); - smxParam.setDistance(-1000); - return smxParam; } + sentenceConstituencyParseList.clear(); + ConcurrentMap simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap sentiment1 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap sentiment2 = new MapMaker().concurrencyLevel(2).makeMap(); + for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) { + Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); + sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree)); + SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); + SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); + simpleSMXlist.put(simpleSMXlist.size(), predictions); + simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector); + } + for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) { + Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); + sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree)); + SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); + SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree); + score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> { + subtracter *= 25; + return subtracter; + }).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item); + for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) { + double dot = nodeVector.dot(simpleSMX); + double elementSum = nodeVector.kron(simpleSMX).elementSum(); + elementSum = Math.round(elementSum * 100.0) / 100.0; + if (dot < 0.1) { + score += 256; + } + if (elementSum < 0.1 && elementSum > 0.0) { + score += 1300; + } else if (elementSum > 0.1 && elementSum < 1.0) { + score -= 1100; + } else { + score -= 1424; + } + } + } + score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500; + DocumentReaderAndWriter readerAndWriter = classifier.makePlainTextReaderAndWriter(); + List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter); + List classifyRaw2 = classifier.classifyRaw(str1, readerAndWriter); + score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200; + int mainSentiment1 = 0; + int longest1 = 0; + int mainSentiment2 = 0; + int longest2 = 0; + for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) { + Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); + int sentiment = RNNCoreAnnotations.getPredictedClass(tree); + String partText = sentence.toString(); + SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); + if (partText.length() > longest1) { + mainSentiment1 = sentiment; + longest1 = partText.length(); + } + } + for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) { + Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class); + int sentiment = RNNCoreAnnotations.getPredictedClass(tree); + SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree); + String partText = sentence.toString(); + if (partText.length() > longest2) { + mainSentiment2 = sentiment; + longest2 = partText.length(); + } + } + if (longest1 != longest2) { + long deffLongest = longest1 > longest2 ? longest1 : longest2; + long deffshorter = longest1 < longest2 ? longest1 : longest2; + if (deffLongest >= (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) { + score += (deffLongest - deffshorter) * 200; + } else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) { + score += (deffLongest - deffshorter) * 200; + } else { + score -= (deffLongest - deffshorter) * 50; + } + } + List sentences = jmweStrAnnotation1.get(CoreAnnotations.SentencesAnnotation.class); + int tokensCounter1 = 0; + int tokensCounter2 = 0; + int anotatorcounter1 = 0; + int anotatorcounter2 = 0; + int inflectedCounterPositive1 = 0; + int inflectedCounterPositive2 = 0; + int inflectedCounterNegative = 0; + int MarkedContinuousCounter1 = 0; + int MarkedContinuousCounter2 = 0; + int UnmarkedPatternCounter = 0; + ConcurrentMap ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap strTokenStems2 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap strTokenForm1 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap strTokenForm2 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap strTokenGetEntry1 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap strTokenGetEntry2 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap strTokenGetiPart1 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap strTokenGetiPart2 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap strTokenEntryPOS1 = new MapMaker().concurrencyLevel(2).makeMap(); + ConcurrentMap strTokenEntryPOS2 = new MapMaker().concurrencyLevel(2).makeMap(); + for (CoreMap sentence : sentences) { + for (IMWE token : sentence.get(JMWEAnnotation.class)) { + if (token.isInflected()) { + inflectedCounterPositive1++; + } else { + inflectedCounterNegative++; + } + strTokenForm1.put(strTokenForm1.size() + 1, token.getForm()); + strTokenGetEntry1.put(strTokenGetEntry1.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1)); + Collection values = token.getPartMap().values(); + IMWEDesc entry = token.getEntry(); + MarkedContinuousCounter1 += entry.getMarkedContinuous(); + UnmarkedPatternCounter += entry.getUnmarkedPattern(); + for (IMWEDesc.IPart iPart : values) { + strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm()); + } + for (String strPostPrefix : entry.getPOS().getPrefixes()) { + strTokenEntryPOS1.put(strTokenEntryPOS1.size() + 1, strPostPrefix); + } + for (IToken tokens : token.getTokens()) { + ITokenMapTag1.put(ITokenMapTag1.size() + 1, tokens.getTag()); + for (String strtoken : tokens.getStems()) { + strTokenStems1.put(strTokenStems1.size() + 1, strtoken); + } + } + tokensCounter1++; + } + anotatorcounter1++; + } + sentences = jmweStrAnnotation2.get(CoreAnnotations.SentencesAnnotation.class); + for (CoreMap sentence : sentences) { + for (IMWE token : sentence.get(JMWEAnnotation.class)) { + if (token.isInflected()) { + inflectedCounterPositive2++; + } else { + inflectedCounterNegative--; + } + strTokenForm2.put(strTokenForm2.size() + 1, token.getForm()); + strTokenGetEntry2.put(strTokenGetEntry2.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1)); + Collection values = token.getPartMap().values(); + IMWEDesc entry = token.getEntry(); + MarkedContinuousCounter2 += entry.getMarkedContinuous(); + UnmarkedPatternCounter += entry.getUnmarkedPattern(); + for (IMWEDesc.IPart iPart : values) { + strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm()); + } + for (String strPostPrefix : entry.getPOS().getPrefixes()) { + strTokenEntryPOS2.put(strTokenEntryPOS2.size() + 1, strPostPrefix); + } + for (IToken tokens : token.getTokens()) { + ITokenMapTag2.put(ITokenMapTag2.size() + 1, tokens.getTag()); + for (String strtoken : tokens.getStems()) { + strTokenStems2.put(strTokenStems2.size() + 1, strtoken); + } + } + tokensCounter2++; + } + anotatorcounter2++; + } + for (String strTokenPos1 : strTokenEntryPOS1.values()) { + for (String strTokenPos2 : strTokenEntryPOS2.values()) { + if (strTokenPos1.equals(strTokenPos2)) { + score += 500; + } + } + } + score += UnmarkedPatternCounter * 1600; + if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) { + score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500 + : (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500; + } + for (String strTokeniPart1 : strTokenGetiPart1.values()) { + for (String strTokeniPart2 : strTokenGetiPart2.values()) { + if (strTokeniPart1.equals(strTokeniPart2)) { + score += 400; + } + } + } + for (String strTokenEntry1 : strTokenGetEntry1.values()) { + for (String strTokenEntry2 : strTokenGetEntry2.values()) { + if (strTokenEntry1.equals(strTokenEntry2)) { + score += 2500; + } + } + } + for (String strmapTag : ITokenMapTag1.values()) { + for (String strmapTag1 : ITokenMapTag2.values()) { + if (strmapTag.equals(strmapTag1)) { + score += 1450; + } + } + } + for (String strTokenForm1itr1 : strTokenForm1.values()) { + for (String strTokenForm1itr2 : strTokenForm2.values()) { + if (strTokenForm1itr1.equals(strTokenForm1itr2)) { + score += 2600; + } else if (strTokenForm1itr1.contains(strTokenForm1itr2)) { + score += 500; + } + } + } + for (String strTokenStem : strTokenStems1.values()) { + for (String strTokenStem1 : strTokenStems2.values()) { + if (strTokenStem.equals(strTokenStem1)) { + score += 1500; + } + } + } + if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) { + score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650; + } + if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) { + score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550; + } + if (anotatorcounter1 > 1 && anotatorcounter2 > 1) { + score += (anotatorcounter1 + anotatorcounter2) * 400; + } + if (tokensCounter1 > 0 && tokensCounter2 > 0) { + score += (tokensCounter1 + tokensCounter2) * 400; + } else { + score -= tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500; + } + LevenshteinDistance leven = new LevenshteinDistance(str, str1); + int SentenceScoreDiff = leven.computeLevenshteinDistance(); + SentenceScoreDiff *= 15; + score -= SentenceScoreDiff; + System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n"); + smxParam.setDistance(score); return smxParam; } - } diff --git a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java index 3c04eabe..6efa12d0 100644 --- a/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java +++ b/ArtificialAutism/src/main/java/PresentationLayer/DiscordHandler.java @@ -12,8 +12,6 @@ nohup screen -d -m -S nonroot java -Xmx6800M -jar /home/javatests/ArtificialAut screen -ls (number1) screen -X -S (number1) quit */ -//https://discordapp.com/developers/applications/ -//https://github.com/Javacord/Javacord package PresentationLayer; import FunctionLayer.CustomError; @@ -43,11 +41,13 @@ public class DiscordHandler { Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex); } }).start(); - MYSQLDatahandler.shiftReduceParserInitiate(); + MYSQLDatahandler.instance.shiftReduceParserInitiate(); MYSQLDatahandler.instance.instantiateExecutor(); + MYSQLDatahandler.instance.updateStringCache(); + MYSQLDatahandler.instance.instantiateAnnotationMap(); if (MYSQLDatahandler.instance.getstringCacheSize() != 0) { - while (MYSQLDatahandler.instance.getlHMSMXSize() * MYSQLDatahandler.instance.getlHMSMXSize() * 2 - < (MYSQLDatahandler.instance.getstringCacheSize() + while (MYSQLDatahandler.instance.getlHMSMXSize() * MYSQLDatahandler.instance.getlHMSMXSize() * 5 + < (MYSQLDatahandler.instance.getstringCacheSize() * MYSQLDatahandler.instance.getstringCacheSize()) - MYSQLDatahandler.instance.getstringCacheSize()) { MYSQLDatahandler.instance.checkIfUpdateMatrixes(); @@ -62,12 +62,6 @@ public class DiscordHandler { strtest = strtest.substring(9, strtest.length() - 1); boolean channelpermissionsDenied = false; switch (strtest) { - case "Server Area": { - if (!event.getServerTextChannel().get().toString().contains("chat-live")) { - channelpermissionsDenied = true; - } - break; - } case "Public Area": { break; } diff --git a/ArtificialAutism/target/ArtificialAutism-1.0.jar b/ArtificialAutism/target/ArtificialAutism-1.0.jar index b546e6a7..a3d3e067 100644 Binary files a/ArtificialAutism/target/ArtificialAutism-1.0.jar and b/ArtificialAutism/target/ArtificialAutism-1.0.jar differ