since server area is gone it fetches some messages from hlstats now, will increase amount later, only potentially issue left is a nullpointer in getresponsemsg if none within 8 is reached, removed pipelines with annotation caching,
This commit is contained in:
parent
232190d076
commit
b3631ab1a0
@ -0,0 +1,42 @@
|
|||||||
|
/*
|
||||||
|
* To change this license header, choose License Headers in Project Properties.
|
||||||
|
* To change this template file, choose Tools | Templates
|
||||||
|
* and open the template in the editor.
|
||||||
|
*/
|
||||||
|
package DataLayer;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.util.logging.Level;
|
||||||
|
import java.util.logging.Logger;
|
||||||
|
import org.apache.commons.dbcp2.BasicDataSource;
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @author install1
|
||||||
|
*/
|
||||||
|
public class DBCPDataSourceHLstats {
|
||||||
|
|
||||||
|
private static BasicDataSource ds = new BasicDataSource();
|
||||||
|
static {
|
||||||
|
try {
|
||||||
|
ds.setDriver(new com.mysql.cj.jdbc.Driver());
|
||||||
|
ds.setUrl("jdbc:mysql://151.80.230.149:3306/unloze_stats?useLegacyDatetimeCode=false&serverTimezone=UTC");
|
||||||
|
ds.setUsername("unloze_stats");
|
||||||
|
ds.setPassword("R8J8E9Fzmcc7ZfDAGAk7");
|
||||||
|
ds.setMaxTotal(-1);
|
||||||
|
ds.setMinIdle(5);
|
||||||
|
ds.setMaxIdle(-1);
|
||||||
|
ds.setMaxOpenPreparedStatements(100);
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
Logger.getLogger(DBCPDataSource.class.getName()).log(Level.SEVERE, null, ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Connection getConnection() throws SQLException {
|
||||||
|
return ds.getConnection();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DBCPDataSourceHLstats() {
|
||||||
|
}
|
||||||
|
}
|
@ -179,6 +179,24 @@ public class DataMapper {
|
|||||||
return LHMSMX;
|
return LHMSMX;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static ConcurrentMap<Integer, String> getHLstatsMessages() {
|
||||||
|
ConcurrentMap<Integer, String> hlStatsMessages = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
try (Connection l_cCon = DBCPDataSourceHLstats.getConnection()) {
|
||||||
|
String l_sSQL = "SELECT message FROM `hlstats_Events_Chat`";
|
||||||
|
try (PreparedStatement l_pStatement = l_cCon.prepareStatement(l_sSQL, java.sql.ResultSet.TYPE_FORWARD_ONLY,
|
||||||
|
java.sql.ResultSet.CONCUR_READ_ONLY)) {
|
||||||
|
try (ResultSet l_rsSearch = l_pStatement.executeQuery()) {
|
||||||
|
while (l_rsSearch.next()) {
|
||||||
|
hlStatsMessages.put(hlStatsMessages.size() + 1, l_rsSearch.getString(1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex);
|
||||||
|
}
|
||||||
|
return hlStatsMessages;
|
||||||
|
}
|
||||||
|
|
||||||
public static void CloseConnections(PreparedStatement ps, ResultSet rs, Connection con) {
|
public static void CloseConnections(PreparedStatement ps, ResultSet rs, Connection con) {
|
||||||
if (rs != null) {
|
if (rs != null) {
|
||||||
try {
|
try {
|
||||||
@ -202,5 +220,4 @@ public class DataMapper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -5,28 +5,23 @@
|
|||||||
*/
|
*/
|
||||||
package FunctionLayer;
|
package FunctionLayer;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
import java.util.concurrent.Callable;
|
import java.util.concurrent.Callable;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
* @author install1
|
* @author install1
|
||||||
*/
|
*/
|
||||||
public class LevenshteinDistance implements Callable<DistanceObject> {
|
public class LevenshteinDistance implements Callable<Map.Entry<String, Integer>> {
|
||||||
|
|
||||||
private CharSequence lhs;
|
private CharSequence lhs;
|
||||||
private CharSequence rhs;
|
private CharSequence rhs;
|
||||||
private DistanceObject dco;
|
private Map.Entry<String, Integer> distanceEntry;
|
||||||
|
|
||||||
private static int minimum(int a, int b, int c) {
|
private static int minimum(int a, int b, int c) {
|
||||||
return Math.min(Math.min(a, b), c);
|
return Math.min(Math.min(a, b), c);
|
||||||
}
|
}
|
||||||
|
|
||||||
public LevenshteinDistance(CharSequence lhs, CharSequence rhs, DistanceObject dco) {
|
|
||||||
this.lhs = lhs;
|
|
||||||
this.rhs = rhs;
|
|
||||||
this.dco = dco;
|
|
||||||
}
|
|
||||||
|
|
||||||
public LevenshteinDistance(CharSequence lhs, CharSequence rhs) {
|
public LevenshteinDistance(CharSequence lhs, CharSequence rhs) {
|
||||||
this.lhs = lhs;
|
this.lhs = lhs;
|
||||||
this.rhs = rhs;
|
this.rhs = rhs;
|
||||||
@ -52,7 +47,7 @@ public class LevenshteinDistance implements Callable<DistanceObject> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public DistanceObject call() {
|
public Map.Entry<String, Integer> call() {
|
||||||
try {
|
try {
|
||||||
int[][] distance = new int[lhs.length() + 1][rhs.length() + 1];
|
int[][] distance = new int[lhs.length() + 1][rhs.length() + 1];
|
||||||
|
|
||||||
@ -70,12 +65,11 @@ public class LevenshteinDistance implements Callable<DistanceObject> {
|
|||||||
distance[i - 1][j - 1] + ((lhs.charAt(i - 1) == rhs.charAt(j - 1)) ? 0 : 1));
|
distance[i - 1][j - 1] + ((lhs.charAt(i - 1) == rhs.charAt(j - 1)) ? 0 : 1));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
dco.setDistance(distance[lhs.length()][rhs.length()]);
|
distanceEntry.setValue(distance[lhs.length()][rhs.length()]);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
System.out.println("ex msg: " + ex.getMessage() + "\n");
|
System.out.println("ex msg: " + ex.getMessage() + "\n");
|
||||||
dco.setDistance(100);
|
return null;
|
||||||
return dco;
|
|
||||||
}
|
}
|
||||||
return dco;
|
return distanceEntry;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -17,6 +17,7 @@ import edu.stanford.nlp.ling.TaggedWord;
|
|||||||
import edu.stanford.nlp.ling.Word;
|
import edu.stanford.nlp.ling.Word;
|
||||||
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
|
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
|
||||||
import edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser;
|
import edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser;
|
||||||
|
import edu.stanford.nlp.pipeline.Annotation;
|
||||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
||||||
import edu.stanford.nlp.process.DocumentPreprocessor;
|
import edu.stanford.nlp.process.DocumentPreprocessor;
|
||||||
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
|
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
|
||||||
@ -33,11 +34,9 @@ import java.util.Map;
|
|||||||
import java.util.Map.Entry;
|
import java.util.Map.Entry;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
import java.util.Set;
|
|
||||||
import java.util.concurrent.Callable;
|
import java.util.concurrent.Callable;
|
||||||
import java.util.concurrent.ConcurrentMap;
|
import java.util.concurrent.ConcurrentMap;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import java.util.concurrent.ForkJoinPool;
|
import java.util.concurrent.ForkJoinPool;
|
||||||
import java.util.concurrent.Future;
|
import java.util.concurrent.Future;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
@ -57,17 +56,20 @@ public class MYSQLDatahandler {
|
|||||||
private volatile boolean refreshMatrixFromDB;
|
private volatile boolean refreshMatrixFromDB;
|
||||||
private static int secondaryIterator = 0;
|
private static int secondaryIterator = 0;
|
||||||
private final ConcurrentMap<Integer, String> stringCache;
|
private final ConcurrentMap<Integer, String> stringCache;
|
||||||
|
private static ConcurrentMap<String, Annotation> pipelineAnnotationCache;
|
||||||
|
private static ConcurrentMap<String, Annotation> pipelineSentimentAnnotationCache;
|
||||||
|
private static ConcurrentMap<String, Annotation> jmweAnnotationCache;
|
||||||
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
|
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
|
||||||
private final Stopwatch stopwatch;
|
private final Stopwatch stopwatch;
|
||||||
private final Stopwatch stopwatch1;
|
private final Stopwatch stopwatch1;
|
||||||
private ExecutorService executor;
|
private ForkJoinPool executor;
|
||||||
|
|
||||||
private static String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz";
|
private static String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz";
|
||||||
private static String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz";
|
private static String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz";
|
||||||
private static String lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz";
|
private static String lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz";
|
||||||
private static String taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger";
|
private static String taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger";
|
||||||
private static String nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz";
|
private static String nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz";
|
||||||
private static String jmweIndexData = "/home/javatests/lib/mweindex_wordnet3.0_semcor1.6.data"; // ./lib/mweindex_wordnet3.0_semcor1.6.data
|
private static String jmweIndexData = "/home/javatests/lib/mweindex_wordnet3.0_semcor1.6.data"; // ./lib/mweindex_wordnet3.0_semcor1.6.data
|
||||||
|
private static String jmweIndexDataLocalTest = "E:/java8/Projects/mweindex_wordnet3.0_semcor1.6.data";
|
||||||
private static MaxentTagger tagger;
|
private static MaxentTagger tagger;
|
||||||
private static ShiftReduceParser model;
|
private static ShiftReduceParser model;
|
||||||
private static String[] options = {"-maxLength", "100"};
|
private static String[] options = {"-maxLength", "100"};
|
||||||
@ -83,27 +85,17 @@ public class MYSQLDatahandler {
|
|||||||
private static StanfordCoreNLP pipelineSentiment;
|
private static StanfordCoreNLP pipelineSentiment;
|
||||||
private static StanfordCoreNLP pipelineJMWE;
|
private static StanfordCoreNLP pipelineJMWE;
|
||||||
|
|
||||||
public static AbstractSequenceClassifier<CoreLabel> getClassifier() {
|
|
||||||
return classifier;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void setClassifier(AbstractSequenceClassifier<CoreLabel> classifier) {
|
|
||||||
MYSQLDatahandler.classifier = classifier;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void instantiateExecutor() {
|
|
||||||
this.executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(),
|
|
||||||
ForkJoinPool.defaultForkJoinWorkerThreadFactory,
|
|
||||||
null, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
public MYSQLDatahandler() {
|
public MYSQLDatahandler() {
|
||||||
this.stopwatch = Stopwatch.createUnstarted();
|
this.stopwatch = Stopwatch.createUnstarted();
|
||||||
this.stopwatch1 = Stopwatch.createStarted();
|
this.stopwatch1 = Stopwatch.createStarted();
|
||||||
this.stringCache = new MapMaker().concurrencyLevel(2).makeMap();
|
this.stringCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
//cant sadly just have one annotation for every pipelines, one pipeline per annotation is required
|
||||||
|
this.jmweAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
this.pipelineAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
this.pipelineSentimentAnnotationCache = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void shiftReduceParserInitiate() {
|
public void shiftReduceParserInitiate() {
|
||||||
try {
|
try {
|
||||||
classifier = CRFClassifier.getClassifierNoExceptions(nerModel);
|
classifier = CRFClassifier.getClassifierNoExceptions(nerModel);
|
||||||
} catch (ClassCastException ex) {
|
} catch (ClassCastException ex) {
|
||||||
@ -127,7 +119,7 @@ public class MYSQLDatahandler {
|
|||||||
propsJMWE.setProperty("customAnnotatorClass.jmwe", "edu.stanford.nlp.pipeline.JMWEAnnotator");
|
propsJMWE.setProperty("customAnnotatorClass.jmwe", "edu.stanford.nlp.pipeline.JMWEAnnotator");
|
||||||
propsJMWE.setProperty("customAnnotatorClass.jmwe.verbose", "false");
|
propsJMWE.setProperty("customAnnotatorClass.jmwe.verbose", "false");
|
||||||
propsJMWE.setProperty("customAnnotatorClass.jmwe.underscoreReplacement", "-");
|
propsJMWE.setProperty("customAnnotatorClass.jmwe.underscoreReplacement", "-");
|
||||||
propsJMWE.setProperty("customAnnotatorClass.jmwe.indexData", jmweIndexData);
|
propsJMWE.setProperty("customAnnotatorClass.jmwe.indexData", jmweIndexData); //jmweIndexDataLocalTest jmweIndexData
|
||||||
propsJMWE.setProperty("customAnnotatorClass.jmwe.detector", "Exhaustive");
|
propsJMWE.setProperty("customAnnotatorClass.jmwe.detector", "Exhaustive");
|
||||||
//"Consecutive", "Exhaustive", "ProperNouns", "Complex" and "CompositeConsecutiveProperNouns"
|
//"Consecutive", "Exhaustive", "ProperNouns", "Complex" and "CompositeConsecutiveProperNouns"
|
||||||
propsJMWE.setProperty("annotators", "tokenize, ssplit, pos, lemma, jmwe");
|
propsJMWE.setProperty("annotators", "tokenize, ssplit, pos, lemma, jmwe");
|
||||||
@ -137,8 +129,26 @@ public class MYSQLDatahandler {
|
|||||||
pipelineJMWE = new StanfordCoreNLP(propsJMWE);
|
pipelineJMWE = new StanfordCoreNLP(propsJMWE);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static StanfordCoreNLP getPipelineJMWE() {
|
public static AbstractSequenceClassifier<CoreLabel> getClassifier() {
|
||||||
return pipelineJMWE;
|
return classifier;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void setClassifier(AbstractSequenceClassifier<CoreLabel> classifier) {
|
||||||
|
MYSQLDatahandler.classifier = classifier;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void updateStringCache() {
|
||||||
|
try {
|
||||||
|
checkIfUpdateStrings();
|
||||||
|
} catch (CustomError ex) {
|
||||||
|
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void instantiateExecutor() {
|
||||||
|
this.executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(),
|
||||||
|
ForkJoinPool.defaultForkJoinWorkerThreadFactory,
|
||||||
|
null, false); //true
|
||||||
}
|
}
|
||||||
|
|
||||||
public static GrammaticalStructureFactory getGsf() {
|
public static GrammaticalStructureFactory getGsf() {
|
||||||
@ -177,6 +187,7 @@ public class MYSQLDatahandler {
|
|||||||
try {
|
try {
|
||||||
DataMapper.createTables();
|
DataMapper.createTables();
|
||||||
stringCache.putAll(getCache());
|
stringCache.putAll(getCache());
|
||||||
|
addHLstatsMessages();
|
||||||
lHMSMX = DataMapper.getAllRelationScores();
|
lHMSMX = DataMapper.getAllRelationScores();
|
||||||
} catch (CustomError ex) {
|
} catch (CustomError ex) {
|
||||||
Logger.getLogger(MYSQLDatahandler.class
|
Logger.getLogger(MYSQLDatahandler.class
|
||||||
@ -184,6 +195,52 @@ public class MYSQLDatahandler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void addHLstatsMessages() {
|
||||||
|
ConcurrentMap<Integer, String> hlStatsMessages = DataMapper.getHLstatsMessages();
|
||||||
|
ConcurrentMap<Integer, String> strCacheLocal = stringCache;
|
||||||
|
int hardcap = 300;
|
||||||
|
int counter = 0;
|
||||||
|
for (String str : hlStatsMessages.values()) {
|
||||||
|
if (!str.startsWith("!")) {
|
||||||
|
boolean present = false;
|
||||||
|
for (String str1 : strCacheLocal.values()) {
|
||||||
|
if (str.equals(str1)) {
|
||||||
|
present = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!present) {
|
||||||
|
MessageResponseHandler.getMessage(str);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (counter >= hardcap) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
counter++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void instantiateAnnotationMap() {
|
||||||
|
for (String str : stringCache.values()) {
|
||||||
|
System.out.println("str annotation jmwe: " + str + "\n");
|
||||||
|
Annotation strAnno = new Annotation(str);
|
||||||
|
pipelineJMWE.annotate(strAnno);
|
||||||
|
jmweAnnotationCache.put(str, strAnno);
|
||||||
|
}
|
||||||
|
for (String str : stringCache.values()) {
|
||||||
|
System.out.println("str annotation pipeline: " + str + "\n");
|
||||||
|
Annotation strAnno = new Annotation(str);
|
||||||
|
pipeline.annotate(strAnno);
|
||||||
|
pipelineAnnotationCache.put(str, strAnno);
|
||||||
|
}
|
||||||
|
for (String str : stringCache.values()) {
|
||||||
|
System.out.println("str annotation pipelinesentiment: " + str + "\n");
|
||||||
|
Annotation strAnno = new Annotation(str);
|
||||||
|
pipelineSentiment.annotate(strAnno);
|
||||||
|
pipelineSentimentAnnotationCache.put(str, strAnno);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public synchronized void checkIfUpdateMatrixes() {
|
public synchronized void checkIfUpdateMatrixes() {
|
||||||
refreshMatrixFromDB = false;
|
refreshMatrixFromDB = false;
|
||||||
int counter = 0;
|
int counter = 0;
|
||||||
@ -233,47 +290,52 @@ public class MYSQLDatahandler {
|
|||||||
boolean present = false;
|
boolean present = false;
|
||||||
LinkedHashMap<String, Double> orDefault = lHMSMX.getOrDefault(str, null);
|
LinkedHashMap<String, Double> orDefault = lHMSMX.getOrDefault(str, null);
|
||||||
if (orDefault != null) {
|
if (orDefault != null) {
|
||||||
Double orDefault1 = orDefault.getOrDefault(str1, null);
|
for (String strkey : orDefault.keySet()) {
|
||||||
if (orDefault1 != null) {
|
if (strkey.equals(str1)) {
|
||||||
// System.out.println("present true \nstr: " + str + "\nstr1: " + str1 + "\n");
|
present = true;
|
||||||
present = true;
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!present) {
|
if (!present) {
|
||||||
orDefault = lHMSMX.getOrDefault(str1, null);
|
orDefault = lHMSMX.getOrDefault(str1, null);
|
||||||
if (orDefault != null) {
|
if (orDefault != null) {
|
||||||
Double orDefault1 = orDefault.getOrDefault(str, null);
|
for (String strkey : orDefault.keySet()) {
|
||||||
if (orDefault1 != null) {
|
if (strkey.equals(str)) {
|
||||||
// System.out.println("present true \nstr: " + str + "\nstr1: " + str1 + "\n");
|
present = true;
|
||||||
present = true;
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!present) {
|
if (!present) {
|
||||||
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
|
|
||||||
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX);
|
|
||||||
futures.put(futures.size() + 1, executor.submit(worker));
|
|
||||||
LinkedHashMap<String, Double> orDefault1 = lHMSMX.getOrDefault(str, null);
|
LinkedHashMap<String, Double> orDefault1 = lHMSMX.getOrDefault(str, null);
|
||||||
if (orDefault1 == null) {
|
if (orDefault1 == null) {
|
||||||
orDefault1 = new LinkedHashMap<String, Double>();
|
orDefault1 = new LinkedHashMap<String, Double>();
|
||||||
}
|
}
|
||||||
orDefault1.put(str1, 0.0);
|
orDefault1.put(str1, 0.0);
|
||||||
lHMSMX.put(str, orDefault1);
|
lHMSMX.put(str, orDefault1);
|
||||||
|
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
|
||||||
|
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX, jmweAnnotationCache.get(str),
|
||||||
|
jmweAnnotationCache.get(str1), pipelineAnnotationCache.get(str), pipelineAnnotationCache.get(str1),
|
||||||
|
pipelineSentimentAnnotationCache.get(str), pipelineSentimentAnnotationCache.get(str1));
|
||||||
|
futures.put(futures.size() + 1, executor.submit(worker));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
|
System.out.println("finished worker assignment, futures size: " + futures.size() + "\n");
|
||||||
for (Future<SimilarityMatrix> future : futures.values()) {
|
for (Future<SimilarityMatrix> future : futures.values()) {
|
||||||
System.out.println("counter: " + counter + "\n");
|
|
||||||
counter++;
|
|
||||||
SimilarityMatrix SMX = new SimilarityMatrix("", "");
|
SimilarityMatrix SMX = new SimilarityMatrix("", "");
|
||||||
try {
|
try {
|
||||||
SMX = future.get(20, TimeUnit.SECONDS);
|
SMX = future.get(20, TimeUnit.SECONDS);
|
||||||
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
||||||
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
|
Logger.getLogger(MYSQLDatahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||||
|
SMX = null;
|
||||||
}
|
}
|
||||||
LinkedHashMap<String, Double> getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
|
System.out.println("counter: " + counter + "\nSMX: " + SMX == null ? "SMX null\n" : "SMX not null\n");
|
||||||
if (getFuture != null) {
|
counter++;
|
||||||
|
if (SMX != null) {
|
||||||
|
LinkedHashMap<String, Double> getFuture = lHMSMX.getOrDefault(SMX.getPrimaryString(), null);
|
||||||
getFuture.put(SMX.getSecondaryString(), SMX.getDistance());
|
getFuture.put(SMX.getSecondaryString(), SMX.getDistance());
|
||||||
lHMSMX.put(SMX.getPrimaryString(), getFuture);
|
lHMSMX.put(SMX.getPrimaryString(), getFuture);
|
||||||
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
|
matrixUpdateList.put(matrixUpdateList.size() + 1, SMX);
|
||||||
@ -309,6 +371,15 @@ public class MYSQLDatahandler {
|
|||||||
for (String str1 : str.values()) {
|
for (String str1 : str.values()) {
|
||||||
stringCache.put(j, str1);
|
stringCache.put(j, str1);
|
||||||
j++;
|
j++;
|
||||||
|
Annotation strAnno = new Annotation(str1);
|
||||||
|
pipelineJMWE.annotate(strAnno);
|
||||||
|
jmweAnnotationCache.put(str1, strAnno);
|
||||||
|
strAnno = new Annotation(str1);
|
||||||
|
pipeline.annotate(strAnno);
|
||||||
|
pipelineAnnotationCache.put(str1, strAnno);
|
||||||
|
strAnno = new Annotation(str1);
|
||||||
|
pipelineSentiment.annotate(strAnno);
|
||||||
|
pipelineSentimentAnnotationCache.put(str1, strAnno);
|
||||||
}
|
}
|
||||||
if (!stopwatch.isRunning()) {
|
if (!stopwatch.isRunning()) {
|
||||||
stopwatch.start();
|
stopwatch.start();
|
||||||
@ -329,7 +400,7 @@ public class MYSQLDatahandler {
|
|||||||
SimilarityMatrix SMXreturn = new SimilarityMatrix("", "");
|
SimilarityMatrix SMXreturn = new SimilarityMatrix("", "");
|
||||||
System.out.println("pre mostSimilarSTR \n");
|
System.out.println("pre mostSimilarSTR \n");
|
||||||
String mostSimilarSTR = mostSimilar(str, strArrs);
|
String mostSimilarSTR = mostSimilar(str, strArrs);
|
||||||
if (mostSimilarSTR != null && !mostSimilarSTR.isEmpty()) {
|
if (mostSimilarSTR != null) {
|
||||||
System.out.println("mostSimilarSTR; " + mostSimilarSTR + "\n");
|
System.out.println("mostSimilarSTR; " + mostSimilarSTR + "\n");
|
||||||
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.getOrDefault(mostSimilarSTR, null);
|
LinkedHashMap<String, Double> orDefault = LHMSMXLocal.getOrDefault(mostSimilarSTR, null);
|
||||||
if (orDefault != null) {
|
if (orDefault != null) {
|
||||||
@ -366,14 +437,24 @@ public class MYSQLDatahandler {
|
|||||||
ConcurrentMap<Integer, Future<SimilarityMatrix>> futureslocal = new MapMaker().concurrencyLevel(2).makeMap();
|
ConcurrentMap<Integer, Future<SimilarityMatrix>> futureslocal = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
for (String str1 : strCache.values()) {
|
for (String str1 : strCache.values()) {
|
||||||
if (!str.equals(str1)) {
|
if (!str.equals(str1)) {
|
||||||
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, new SimilarityMatrix(str, str1));
|
SimilarityMatrix SMX = new SimilarityMatrix(str, str1);
|
||||||
|
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, SMX,
|
||||||
|
jmweAnnotationCache.get(str), jmweAnnotationCache.get(str1), pipelineAnnotationCache.get(str),
|
||||||
|
pipelineAnnotationCache.get(str1), pipelineSentimentAnnotationCache.get(str),
|
||||||
|
pipelineSentimentAnnotationCache.get(str1));
|
||||||
futureslocal.put(futureslocal.size() + 1, executor.submit(worker));
|
futureslocal.put(futureslocal.size() + 1, executor.submit(worker));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
int index = 0;
|
int index = 0;
|
||||||
for (Future<SimilarityMatrix> future : futureslocal.values()) {
|
for (Future<SimilarityMatrix> future : futureslocal.values()) {
|
||||||
|
SimilarityMatrix SMX = new SimilarityMatrix("", "");
|
||||||
try {
|
try {
|
||||||
SimilarityMatrix SMX = future.get((long) 20, TimeUnit.SECONDS);
|
SMX = future.get(20, TimeUnit.SECONDS);
|
||||||
|
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
||||||
|
System.out.println("ex getResponsemsg: " + ex.getMessage() + "\n");
|
||||||
|
SMX = null;
|
||||||
|
}
|
||||||
|
if (SMX != null) {
|
||||||
double distance = SMX.getDistance();
|
double distance = SMX.getDistance();
|
||||||
System.out.println("index: " + index + "\nfutures size: " + futureslocal.values().size() + "\nScore: " + SMX.getDistance() + "\nSecondary: "
|
System.out.println("index: " + index + "\nfutures size: " + futureslocal.values().size() + "\nScore: " + SMX.getDistance() + "\nSecondary: "
|
||||||
+ SMX.getSecondaryString() + "\nPrimary: " + SMX.getPrimaryString() + "\n");
|
+ SMX.getSecondaryString() + "\nPrimary: " + SMX.getPrimaryString() + "\n");
|
||||||
@ -381,8 +462,6 @@ public class MYSQLDatahandler {
|
|||||||
Score = distance;
|
Score = distance;
|
||||||
SMXreturn = SMX;
|
SMXreturn = SMX;
|
||||||
}
|
}
|
||||||
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
|
||||||
System.out.println("ex: " + ex.getMessage() + "\n");
|
|
||||||
}
|
}
|
||||||
index++;
|
index++;
|
||||||
}
|
}
|
||||||
@ -394,18 +473,19 @@ public class MYSQLDatahandler {
|
|||||||
public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings) {
|
public String mostSimilar(String toBeCompared, ConcurrentMap<Integer, String> concurrentStrings) {
|
||||||
int minDistance = 8;
|
int minDistance = 8;
|
||||||
String similar = "";
|
String similar = "";
|
||||||
ConcurrentMap<Integer, Future<DistanceObject>> futures = new MapMaker().concurrencyLevel(2).makeMap();
|
ConcurrentMap<Integer, Future<Entry<String, Integer>>> futures = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
concurrentStrings.values().stream().map((str) -> new LevenshteinDistance(str, toBeCompared, new DistanceObject())).forEachOrdered((worker) -> {
|
for (String str : concurrentStrings.values()) {
|
||||||
|
Callable<Entry<String, Integer>> worker = new LevenshteinDistance(toBeCompared, str);
|
||||||
futures.put(futures.size() + 1, executor.submit(worker));
|
futures.put(futures.size() + 1, executor.submit(worker));
|
||||||
});
|
}
|
||||||
for (Future<DistanceObject> future : futures.values()) {
|
for (Future<Entry<String, Integer>> future : futures.values()) {
|
||||||
try {
|
try {
|
||||||
DistanceObject d = future.get();
|
Entry<String, Integer> futureEntry = future.get();
|
||||||
int distance = d.getDistance();
|
int distance = futureEntry.getValue();
|
||||||
System.out.println("distance: " + distance + "\n");
|
|
||||||
if (distance < minDistance) {
|
if (distance < minDistance) {
|
||||||
|
System.out.println("distance: " + distance + "\n");
|
||||||
minDistance = distance;
|
minDistance = distance;
|
||||||
similar = d.getSentence();
|
similar = futureEntry.getKey();
|
||||||
}
|
}
|
||||||
} catch (NullPointerException | InterruptedException | ExecutionException ex) {
|
} catch (NullPointerException | InterruptedException | ExecutionException ex) {
|
||||||
System.out.println("failed future\n");
|
System.out.println("failed future\n");
|
||||||
@ -605,17 +685,10 @@ public class MYSQLDatahandler {
|
|||||||
ConcurrentMap<Integer, String> returnmap = new MapMaker().concurrencyLevel(2).makeMap();
|
ConcurrentMap<Integer, String> returnmap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
ConcurrentMap<Integer, String> allStrings = stringCache;
|
ConcurrentMap<Integer, String> allStrings = stringCache;
|
||||||
int intervalAprove = allStrings.values().size() / 10;
|
int intervalAprove = allStrings.values().size() / 10;
|
||||||
String str1 = "and to revise the questions and materials and such";
|
|
||||||
String str2 = "as William said earlier. Visiting your dentist once or twice a year is enough";
|
|
||||||
String str3 = "At least, you have more time to prepare then";
|
|
||||||
for (String str : strmap.values()) {
|
for (String str : strmap.values()) {
|
||||||
|
int counter = 0;
|
||||||
|
boolean calculationIssues = false;
|
||||||
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
|
ConcurrentMap<Integer, Future<SimilarityMatrix>> futures = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, str1, new SimilarityMatrix(str, str1));
|
|
||||||
futures.put(futures.size() + 1, executor.submit(worker));
|
|
||||||
worker = new SentimentAnalyzerTest(str, str2, new SimilarityMatrix(str, str2));
|
|
||||||
futures.put(futures.size() + 1, executor.submit(worker));
|
|
||||||
worker = new SentimentAnalyzerTest(str, str3, new SimilarityMatrix(str, str3));
|
|
||||||
futures.put(futures.size() + 1, executor.submit(worker));
|
|
||||||
int ij2 = 0;
|
int ij2 = 0;
|
||||||
if (allStrings.isEmpty()) {
|
if (allStrings.isEmpty()) {
|
||||||
allStrings = strmap;
|
allStrings = strmap;
|
||||||
@ -624,15 +697,24 @@ public class MYSQLDatahandler {
|
|||||||
if (ij2 > intervalAprove) {
|
if (ij2 > intervalAprove) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
worker = new SentimentAnalyzerTest(str, strValues, new SimilarityMatrix(str, strValues));
|
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, strValues, new SimilarityMatrix(str, strValues),
|
||||||
|
jmweAnnotationCache.get(str), jmweAnnotationCache.get(strValues), pipelineAnnotationCache.get(str),
|
||||||
|
pipelineAnnotationCache.get(strValues), pipelineSentimentAnnotationCache.get(str),
|
||||||
|
pipelineSentimentAnnotationCache.get(strValues));
|
||||||
futures.put(futures.size() + 1, executor.submit(worker));
|
futures.put(futures.size() + 1, executor.submit(worker));
|
||||||
ij2++;
|
ij2++;
|
||||||
}
|
}
|
||||||
int counter = 0;
|
|
||||||
boolean calculationIssues = false;
|
|
||||||
for (Future<SimilarityMatrix> future : futures.values()) {
|
for (Future<SimilarityMatrix> future : futures.values()) {
|
||||||
try {
|
try {
|
||||||
future.get(20, TimeUnit.SECONDS);
|
SimilarityMatrix get = future.get(20, TimeUnit.SECONDS);
|
||||||
|
if (get == null) {
|
||||||
|
System.out.println("counter timeouts: " + counter + "\n");
|
||||||
|
counter++;
|
||||||
|
if (counter >= 10) {
|
||||||
|
calculationIssues = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
} catch (InterruptedException | ExecutionException | TimeoutException ex) {
|
||||||
System.out.println("counter timeouts: " + counter + "\n");
|
System.out.println("counter timeouts: " + counter + "\n");
|
||||||
counter++;
|
counter++;
|
||||||
|
@ -41,49 +41,6 @@ public class SimilarityMatrix{
|
|||||||
this.distance = result;
|
this.distance = result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
public double getDistanceCalculations() {
|
|
||||||
ILexicalDatabase db = new NictWordNet();
|
|
||||||
WS4JConfiguration.getInstance().setMFS(true);
|
|
||||||
RelatednessCalculator rc1 = new WuPalmer(db);
|
|
||||||
RelatednessCalculator rc2 = new Resnik(db);
|
|
||||||
RelatednessCalculator rc3 = new JiangConrath(db);
|
|
||||||
RelatednessCalculator rc4 = new Lin(db);
|
|
||||||
RelatednessCalculator rc5 = new LeacockChodorow(db);
|
|
||||||
RelatednessCalculator rc6 = new Path(db);
|
|
||||||
RelatednessCalculator rc7 = new Lesk(db);
|
|
||||||
RelatednessCalculator rc8 = new HirstStOnge(db);
|
|
||||||
double maxScore = -1D;
|
|
||||||
List<RelatednessCalculator> RCList = new ArrayList();
|
|
||||||
RCList.add(rc1);
|
|
||||||
RCList.add(rc2);
|
|
||||||
RCList.add(rc3);
|
|
||||||
RCList.add(rc4);
|
|
||||||
RCList.add(rc5);
|
|
||||||
RCList.add(rc6);
|
|
||||||
RCList.add(rc7);
|
|
||||||
RCList.add(rc8);
|
|
||||||
for (RelatednessCalculator rc : RCList) {
|
|
||||||
double s = rc.calcRelatednessOfWords(PrimaryString, SecondaryString);
|
|
||||||
s /= 1000;
|
|
||||||
if (s > 0.000000) {
|
|
||||||
System.out.println("s: " + s + "\n" + " PrimaryString: " + PrimaryString + "\n" + " SecondaryString: " + SecondaryString + "\n"
|
|
||||||
+ " rc: " + rc.toString() + "\n");
|
|
||||||
}
|
|
||||||
String str = String.format("%.12f", s);
|
|
||||||
if (str.contains(",")) {
|
|
||||||
str = str.substring(0, str.indexOf(","));
|
|
||||||
}
|
|
||||||
int strend = str.length() > 6 ? 6 : str.length();
|
|
||||||
str = str.substring(0, strend);
|
|
||||||
double score = Double.valueOf(str);
|
|
||||||
if (score > maxScore) {
|
|
||||||
maxScore = score;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return maxScore == -1D ? 0.00 : maxScore;
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
public String getPrimaryString() {
|
public String getPrimaryString() {
|
||||||
return PrimaryString;
|
return PrimaryString;
|
||||||
}
|
}
|
||||||
|
@ -53,7 +53,6 @@ import org.ejml.simple.SimpleMatrix;
|
|||||||
public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
||||||
|
|
||||||
private SimilarityMatrix smxParam;
|
private SimilarityMatrix smxParam;
|
||||||
|
|
||||||
private String str;
|
private String str;
|
||||||
private String str1;
|
private String str1;
|
||||||
private ShiftReduceParser model;
|
private ShiftReduceParser model;
|
||||||
@ -61,372 +60,364 @@ public class SentimentAnalyzerTest implements Callable<SimilarityMatrix> {
|
|||||||
private GrammaticalStructureFactory gsf;
|
private GrammaticalStructureFactory gsf;
|
||||||
private StanfordCoreNLP pipeline;
|
private StanfordCoreNLP pipeline;
|
||||||
private StanfordCoreNLP pipelineSentiment;
|
private StanfordCoreNLP pipelineSentiment;
|
||||||
private StanfordCoreNLP pipelineJMWE;
|
|
||||||
private AbstractSequenceClassifier classifier;
|
private AbstractSequenceClassifier classifier;
|
||||||
|
private Annotation jmweStrAnnotation1;
|
||||||
|
private Annotation jmweStrAnnotation2;
|
||||||
|
private Annotation pipelineAnnotation1;
|
||||||
|
private Annotation pipelineAnnotation2;
|
||||||
|
private Annotation pipelineAnnotation1Sentiment;
|
||||||
|
private Annotation pipelineAnnotation2Sentiment;
|
||||||
|
|
||||||
public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam) {
|
public SentimentAnalyzerTest(String str, String str1, SimilarityMatrix smxParam, Annotation str1Annotation, Annotation str2Annotation,
|
||||||
|
Annotation strPipeline1, Annotation strPipeline2, Annotation strPipeSentiment1, Annotation strPipeSentiment2) {
|
||||||
this.str = str;
|
this.str = str;
|
||||||
this.str1 = str1;
|
this.str1 = str1;
|
||||||
this.smxParam = smxParam;
|
this.smxParam = smxParam;
|
||||||
model = MYSQLDatahandler.getModel();
|
this.model = MYSQLDatahandler.getModel();
|
||||||
tagger = MYSQLDatahandler.getTagger();
|
this.tagger = MYSQLDatahandler.getTagger();
|
||||||
pipeline = MYSQLDatahandler.getPipeline();
|
this.pipeline = MYSQLDatahandler.getPipeline();
|
||||||
pipelineSentiment = MYSQLDatahandler.getPipelineSentiment();
|
this.pipelineSentiment = MYSQLDatahandler.getPipelineSentiment();
|
||||||
pipelineJMWE = MYSQLDatahandler.getPipelineJMWE();
|
this.gsf = MYSQLDatahandler.getGsf();
|
||||||
gsf = MYSQLDatahandler.getGsf();
|
this.classifier = MYSQLDatahandler.getClassifier();
|
||||||
classifier = MYSQLDatahandler.getClassifier();
|
this.jmweStrAnnotation1 = str1Annotation;
|
||||||
|
this.jmweStrAnnotation2 = str2Annotation;
|
||||||
|
this.pipelineAnnotation1 = strPipeline1;
|
||||||
|
this.pipelineAnnotation2 = strPipeline2;
|
||||||
|
this.pipelineAnnotation1Sentiment = strPipeSentiment1; //maybe process?
|
||||||
|
this.pipelineAnnotation2Sentiment = strPipeSentiment2;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SimilarityMatrix call() {
|
public SimilarityMatrix call() {
|
||||||
try {
|
Double score = -100.0;
|
||||||
Double score = -100.0;
|
List<List<TaggedWord>> taggedwordlist1 = new ArrayList();
|
||||||
List<List<TaggedWord>> taggedwordlist1 = new ArrayList();
|
List<List<TaggedWord>> taggedwordlist2 = new ArrayList();
|
||||||
List<List<TaggedWord>> taggedwordlist2 = new ArrayList();
|
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
|
||||||
DocumentPreprocessor tokenizer = new DocumentPreprocessor(new StringReader(str1));
|
for (List<HasWord> sentence : tokenizer) {
|
||||||
for (List<HasWord> sentence : tokenizer) {
|
taggedwordlist1.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
||||||
taggedwordlist1.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
}
|
||||||
}
|
tokenizer = new DocumentPreprocessor(new StringReader(str));
|
||||||
tokenizer = new DocumentPreprocessor(new StringReader(str));
|
for (List<HasWord> sentence : tokenizer) {
|
||||||
for (List<HasWord> sentence : tokenizer) {
|
taggedwordlist2.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
||||||
taggedwordlist2.add(model.apply(tagger.tagSentence(sentence)).taggedYield());
|
}
|
||||||
}
|
int counter = 0;
|
||||||
int counter = 0;
|
int counter1 = 0;
|
||||||
int counter1 = 0;
|
counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum);
|
||||||
counter = taggedwordlist2.stream().map((taggedlist2) -> taggedlist2.size()).reduce(counter, Integer::sum);
|
counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum);
|
||||||
counter1 = taggedwordlist1.stream().map((taggedlist1) -> taggedlist1.size()).reduce(counter1, Integer::sum);
|
int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter;
|
||||||
int overValue = counter >= counter1 ? counter - counter1 : counter1 - counter;
|
overValue *= 16;
|
||||||
overValue *= 16;
|
score -= overValue;
|
||||||
score -= overValue;
|
ConcurrentMap<Integer, String> tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
ConcurrentMap<Integer, String> tgwlistIndex = new MapMaker().concurrencyLevel(2).makeMap();
|
taggedwordlist1.forEach((TGWList) -> {
|
||||||
taggedwordlist1.forEach((TGWList) -> {
|
TGWList.forEach((TaggedWord) -> {
|
||||||
TGWList.forEach((TaggedWord) -> {
|
if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) {
|
||||||
if (!tgwlistIndex.values().contains(TaggedWord.tag()) && !TaggedWord.tag().equals(":")) {
|
tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag());
|
||||||
tgwlistIndex.put(tgwlistIndex.size() + 1, TaggedWord.tag());
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
taggedwordlist1.clear();
|
|
||||||
AtomicInteger runCount = new AtomicInteger(0);
|
|
||||||
taggedwordlist2.forEach((TGWList) -> {
|
|
||||||
TGWList.forEach((TaggedWord) -> {
|
|
||||||
if (tgwlistIndex.values().contains(TaggedWord.tag())) {
|
|
||||||
tgwlistIndex.values().remove(TaggedWord.tag());
|
|
||||||
runCount.getAndIncrement();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
tgwlistIndex.clear();
|
|
||||||
taggedwordlist2.clear();
|
|
||||||
score += runCount.get() * 64;
|
|
||||||
Annotation annotation = new Annotation(str1);
|
|
||||||
pipeline.annotate(annotation);
|
|
||||||
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) {
|
|
||||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
|
||||||
sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse);
|
|
||||||
}
|
|
||||||
Annotation annotation1 = new Annotation(str);
|
|
||||||
pipeline.annotate(annotation1);
|
|
||||||
for (CoreMap sentence : annotation1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
|
||||||
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
|
||||||
GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse);
|
|
||||||
Collection<TypedDependency> allTypedDependencies = gs.allTypedDependencies();
|
|
||||||
ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) {
|
|
||||||
Set<Constituent> inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1);
|
|
||||||
Set<Constituent> inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse);
|
|
||||||
ConcurrentMap<Integer, String> constiLabels = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
for (Constituent consti : inT1notT2) {
|
|
||||||
for (Constituent consti1 : inT2notT1) {
|
|
||||||
if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) {
|
|
||||||
score += 64;
|
|
||||||
constiLabels.put(constiLabels.size(), consti.value());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1);
|
|
||||||
Collection<TypedDependency> allTypedDependencies1 = gs1.allTypedDependencies();
|
|
||||||
for (TypedDependency TDY1 : allTypedDependencies1) {
|
|
||||||
IndexedWord dep = TDY1.dep();
|
|
||||||
IndexedWord gov = TDY1.gov();
|
|
||||||
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
|
|
||||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
|
||||||
score += 900;
|
|
||||||
}
|
|
||||||
GrammaticalRelation reln = TDY1.reln();
|
|
||||||
if (reln.isApplicable(sentenceConstituencyParse)) {
|
|
||||||
score += 256;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (TypedDependency TDY : allTypedDependencies) {
|
|
||||||
IndexedWord dep = TDY.dep();
|
|
||||||
IndexedWord gov = TDY.gov();
|
|
||||||
GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep);
|
|
||||||
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
|
||||||
score += 900;
|
|
||||||
}
|
|
||||||
GrammaticalRelation reln = TDY.reln();
|
|
||||||
if (reln.isApplicable(sentenceConstituencyParse1)) {
|
|
||||||
score += 256;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
AtomicInteger runCount1 = new AtomicInteger(0);
|
|
||||||
sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> {
|
|
||||||
sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma())
|
|
||||||
&& !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> {
|
|
||||||
filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma());
|
|
||||||
return _item;
|
|
||||||
}).forEachOrdered((_item) -> {
|
|
||||||
runCount1.getAndIncrement();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
score += runCount1.get() * 1500;
|
|
||||||
}
|
}
|
||||||
}
|
});
|
||||||
sentenceConstituencyParseList.clear();
|
});
|
||||||
Annotation annotationSentiment1 = pipelineSentiment.process(str);
|
taggedwordlist1.clear();
|
||||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap();
|
AtomicInteger runCount = new AtomicInteger(0);
|
||||||
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap();
|
taggedwordlist2.forEach((TGWList) -> {
|
||||||
ConcurrentMap<Integer, Integer> sentiment1 = new MapMaker().concurrencyLevel(2).makeMap();
|
TGWList.forEach((TaggedWord) -> {
|
||||||
ConcurrentMap<Integer, Integer> sentiment2 = new MapMaker().concurrencyLevel(2).makeMap();
|
if (tgwlistIndex.values().contains(TaggedWord.tag())) {
|
||||||
for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
tgwlistIndex.values().remove(TaggedWord.tag());
|
||||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
runCount.getAndIncrement();
|
||||||
sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree));
|
}
|
||||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
});
|
||||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
});
|
||||||
simpleSMXlist.put(simpleSMXlist.size(), predictions);
|
tgwlistIndex.clear();
|
||||||
simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector);
|
taggedwordlist2.clear();
|
||||||
}
|
score += runCount.get() * 64;
|
||||||
annotationSentiment1 = pipelineSentiment.process(str1);
|
ConcurrentMap<Integer, Tree> sentenceConstituencyParseList = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
for (CoreMap sentence : annotationSentiment1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
for (CoreMap sentence : pipelineAnnotation1.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||||
sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree));
|
sentenceConstituencyParseList.put(sentenceConstituencyParseList.size(), sentenceConstituencyParse);
|
||||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
}
|
||||||
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
for (CoreMap sentence : pipelineAnnotation2.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||||
score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> {
|
Tree sentenceConstituencyParse = sentence.get(TreeCoreAnnotations.TreeAnnotation.class);
|
||||||
subtracter *= 25;
|
GrammaticalStructure gs = gsf.newGrammaticalStructure(sentenceConstituencyParse);
|
||||||
return subtracter;
|
Collection<TypedDependency> allTypedDependencies = gs.allTypedDependencies();
|
||||||
}).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item);
|
ConcurrentMap<Integer, String> filerTreeContent = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
|
for (Tree sentenceConstituencyParse1 : sentenceConstituencyParseList.values()) {
|
||||||
double dot = nodeVector.dot(simpleSMX);
|
Set<Constituent> inT1notT2 = Tdiff.markDiff(sentenceConstituencyParse, sentenceConstituencyParse1);
|
||||||
double elementSum = nodeVector.kron(simpleSMX).elementSum();
|
Set<Constituent> inT2notT1 = Tdiff.markDiff(sentenceConstituencyParse1, sentenceConstituencyParse);
|
||||||
elementSum = Math.round(elementSum * 100.0) / 100.0;
|
ConcurrentMap<Integer, String> constiLabels = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
if (dot < 0.1) {
|
for (Constituent consti : inT1notT2) {
|
||||||
|
for (Constituent consti1 : inT2notT1) {
|
||||||
|
if (consti.value().equals(consti1.value()) && !constiLabels.values().contains(consti.value())) {
|
||||||
|
score += 64;
|
||||||
|
constiLabels.put(constiLabels.size(), consti.value());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
GrammaticalStructure gs1 = gsf.newGrammaticalStructure(sentenceConstituencyParse1);
|
||||||
|
Collection<TypedDependency> allTypedDependencies1 = gs1.allTypedDependencies();
|
||||||
|
for (TypedDependency TDY1 : allTypedDependencies1) {
|
||||||
|
IndexedWord dep = TDY1.dep();
|
||||||
|
IndexedWord gov = TDY1.gov();
|
||||||
|
GrammaticalRelation grammaticalRelation = gs.getGrammaticalRelation(gov, dep);
|
||||||
|
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||||
|
score += 900;
|
||||||
|
}
|
||||||
|
GrammaticalRelation reln = TDY1.reln();
|
||||||
|
if (reln.isApplicable(sentenceConstituencyParse)) {
|
||||||
score += 256;
|
score += 256;
|
||||||
}
|
}
|
||||||
if (elementSum < 0.1 && elementSum > 0.0) {
|
}
|
||||||
score += 1300;
|
for (TypedDependency TDY : allTypedDependencies) {
|
||||||
} else if (elementSum > 0.1 && elementSum < 1.0) {
|
IndexedWord dep = TDY.dep();
|
||||||
score -= 1100;
|
IndexedWord gov = TDY.gov();
|
||||||
} else {
|
GrammaticalRelation grammaticalRelation = gs1.getGrammaticalRelation(gov, dep);
|
||||||
score -= 1424;
|
if (grammaticalRelation.isApplicable(sentenceConstituencyParse)) {
|
||||||
|
score += 900;
|
||||||
|
}
|
||||||
|
GrammaticalRelation reln = TDY.reln();
|
||||||
|
if (reln.isApplicable(sentenceConstituencyParse1)) {
|
||||||
|
score += 256;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
AtomicInteger runCount1 = new AtomicInteger(0);
|
||||||
|
sentenceConstituencyParse.taggedLabeledYield().forEach((LBW) -> {
|
||||||
|
sentenceConstituencyParse1.taggedLabeledYield().stream().filter((LBW1) -> (LBW.lemma().equals(LBW1.lemma())
|
||||||
|
&& !filerTreeContent.values().contains(LBW.lemma()))).map((_item) -> {
|
||||||
|
filerTreeContent.put(filerTreeContent.size() + 1, LBW.lemma());
|
||||||
|
return _item;
|
||||||
|
}).forEachOrdered((_item) -> {
|
||||||
|
runCount1.getAndIncrement();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
score += runCount1.get() * 1500;
|
||||||
}
|
}
|
||||||
score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500;
|
|
||||||
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
|
|
||||||
List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter);
|
|
||||||
List classifyRaw2 = classifier.classifyRaw(str1, readerAndWriter);
|
|
||||||
score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200;
|
|
||||||
Annotation annotationSentiment = pipelineSentiment.process(str);
|
|
||||||
int mainSentiment1 = 0;
|
|
||||||
int longest1 = 0;
|
|
||||||
int mainSentiment2 = 0;
|
|
||||||
int longest2 = 0;
|
|
||||||
for (CoreMap sentence : annotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
|
||||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
|
||||||
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
|
||||||
String partText = sentence.toString();
|
|
||||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
|
||||||
if (partText.length() > longest1) {
|
|
||||||
mainSentiment1 = sentiment;
|
|
||||||
longest1 = partText.length();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
annotationSentiment = pipelineSentiment.process(str1);
|
|
||||||
for (CoreMap sentence : annotationSentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
|
||||||
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
|
||||||
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
|
||||||
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
|
||||||
String partText = sentence.toString();
|
|
||||||
if (partText.length() > longest2) {
|
|
||||||
mainSentiment2 = sentiment;
|
|
||||||
longest2 = partText.length();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (longest1 != longest2) {
|
|
||||||
long deffLongest = longest1 > longest2 ? longest1 : longest2;
|
|
||||||
long deffshorter = longest1 < longest2 ? longest1 : longest2;
|
|
||||||
if (deffLongest >= (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
|
|
||||||
score += (deffLongest - deffshorter) * 200;
|
|
||||||
} else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) {
|
|
||||||
score += (deffLongest - deffshorter) * 200;
|
|
||||||
} else {
|
|
||||||
score -= (deffLongest - deffshorter) * 50;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Annotation jmweStrAnnotation = new Annotation(str);
|
|
||||||
pipelineJMWE.annotate(jmweStrAnnotation);
|
|
||||||
List<CoreMap> sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class);
|
|
||||||
int tokensCounter1 = 0;
|
|
||||||
int tokensCounter2 = 0;
|
|
||||||
int anotatorcounter1 = 0;
|
|
||||||
int anotatorcounter2 = 0;
|
|
||||||
int inflectedCounterPositive1 = 0;
|
|
||||||
int inflectedCounterPositive2 = 0;
|
|
||||||
int inflectedCounterNegative = 0;
|
|
||||||
int MarkedContinuousCounter1 = 0;
|
|
||||||
int MarkedContinuousCounter2 = 0;
|
|
||||||
int UnmarkedPatternCounter = 0;
|
|
||||||
ConcurrentMap<Integer, String> ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> strTokenStems2 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> strTokenForm1 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> strTokenForm2 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> strTokenGetEntry1 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> strTokenGetEntry2 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> strTokenGetiPart1 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> strTokenGetiPart2 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> strTokenEntryPOS1 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
ConcurrentMap<Integer, String> strTokenEntryPOS2 = new MapMaker().concurrencyLevel(2).makeMap();
|
|
||||||
for (CoreMap sentence : sentences) {
|
|
||||||
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
|
||||||
if (token.isInflected()) {
|
|
||||||
inflectedCounterPositive1++;
|
|
||||||
} else {
|
|
||||||
inflectedCounterNegative++;
|
|
||||||
}
|
|
||||||
strTokenForm1.put(strTokenForm1.size() + 1, token.getForm());
|
|
||||||
strTokenGetEntry1.put(strTokenGetEntry1.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
|
||||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
|
||||||
IMWEDesc entry = token.getEntry();
|
|
||||||
MarkedContinuousCounter1 += entry.getMarkedContinuous();
|
|
||||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
|
||||||
for (IMWEDesc.IPart iPart : values) {
|
|
||||||
strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm());
|
|
||||||
}
|
|
||||||
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
|
||||||
strTokenEntryPOS1.put(strTokenEntryPOS1.size() + 1, strPostPrefix);
|
|
||||||
}
|
|
||||||
for (IToken tokens : token.getTokens()) {
|
|
||||||
ITokenMapTag1.put(ITokenMapTag1.size() + 1, tokens.getTag());
|
|
||||||
for (String strtoken : tokens.getStems()) {
|
|
||||||
strTokenStems1.put(strTokenStems1.size() + 1, strtoken);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tokensCounter1++;
|
|
||||||
}
|
|
||||||
anotatorcounter1++;
|
|
||||||
}
|
|
||||||
jmweStrAnnotation = new Annotation(str1);
|
|
||||||
pipelineJMWE.annotate(jmweStrAnnotation);
|
|
||||||
sentences = jmweStrAnnotation.get(CoreAnnotations.SentencesAnnotation.class);
|
|
||||||
for (CoreMap sentence : sentences) {
|
|
||||||
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
|
||||||
if (token.isInflected()) {
|
|
||||||
inflectedCounterPositive2++;
|
|
||||||
} else {
|
|
||||||
inflectedCounterNegative--;
|
|
||||||
}
|
|
||||||
strTokenForm2.put(strTokenForm2.size() + 1, token.getForm());
|
|
||||||
strTokenGetEntry2.put(strTokenGetEntry2.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
|
||||||
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
|
||||||
IMWEDesc entry = token.getEntry();
|
|
||||||
MarkedContinuousCounter2 += entry.getMarkedContinuous();
|
|
||||||
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
|
||||||
for (IMWEDesc.IPart iPart : values) {
|
|
||||||
strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm());
|
|
||||||
}
|
|
||||||
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
|
||||||
strTokenEntryPOS2.put(strTokenEntryPOS2.size() + 1, strPostPrefix);
|
|
||||||
}
|
|
||||||
for (IToken tokens : token.getTokens()) {
|
|
||||||
ITokenMapTag2.put(ITokenMapTag2.size() + 1, tokens.getTag());
|
|
||||||
for (String strtoken : tokens.getStems()) {
|
|
||||||
strTokenStems2.put(strTokenStems2.size() + 1, strtoken);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tokensCounter2++;
|
|
||||||
}
|
|
||||||
anotatorcounter2++;
|
|
||||||
}
|
|
||||||
for (String strTokenPos1 : strTokenEntryPOS1.values()) {
|
|
||||||
for (String strTokenPos2 : strTokenEntryPOS2.values()) {
|
|
||||||
if (strTokenPos1.equals(strTokenPos2)) {
|
|
||||||
score += 500;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
score += UnmarkedPatternCounter * 1600;
|
|
||||||
if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) {
|
|
||||||
score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500
|
|
||||||
: (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500;
|
|
||||||
}
|
|
||||||
for (String strTokeniPart1 : strTokenGetiPart1.values()) {
|
|
||||||
for (String strTokeniPart2 : strTokenGetiPart2.values()) {
|
|
||||||
if (strTokeniPart1.equals(strTokeniPart2)) {
|
|
||||||
score += 400;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (String strTokenEntry1 : strTokenGetEntry1.values()) {
|
|
||||||
for (String strTokenEntry2 : strTokenGetEntry2.values()) {
|
|
||||||
if (strTokenEntry1.equals(strTokenEntry2)) {
|
|
||||||
score += 2500;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (String strmapTag : ITokenMapTag1.values()) {
|
|
||||||
for (String strmapTag1 : ITokenMapTag2.values()) {
|
|
||||||
if (strmapTag.equals(strmapTag1)) {
|
|
||||||
score += 1450;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (String strTokenForm1itr1 : strTokenForm1.values()) {
|
|
||||||
for (String strTokenForm1itr2 : strTokenForm2.values()) {
|
|
||||||
if (strTokenForm1itr1.equals(strTokenForm1itr2)) {
|
|
||||||
score += 2600;
|
|
||||||
} else if (strTokenForm1itr1.contains(strTokenForm1itr2)) {
|
|
||||||
score += 500;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (String strTokenStem : strTokenStems1.values()) {
|
|
||||||
for (String strTokenStem1 : strTokenStems2.values()) {
|
|
||||||
if (strTokenStem.equals(strTokenStem1)) {
|
|
||||||
score += 1500;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) {
|
|
||||||
score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650;
|
|
||||||
}
|
|
||||||
if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) {
|
|
||||||
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
|
|
||||||
}
|
|
||||||
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
|
|
||||||
score += (anotatorcounter1 + anotatorcounter2) * 400;
|
|
||||||
}
|
|
||||||
if (tokensCounter1 > 0 && tokensCounter2 > 0) {
|
|
||||||
score += (tokensCounter1 + tokensCounter2) * 400;
|
|
||||||
} else {
|
|
||||||
score -= tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
|
||||||
}
|
|
||||||
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
|
|
||||||
int SentenceScoreDiff = leven.computeLevenshteinDistance();
|
|
||||||
SentenceScoreDiff *= 15;
|
|
||||||
score -= SentenceScoreDiff;
|
|
||||||
System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n");
|
|
||||||
smxParam.setDistance(score);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
System.out.println("ex: " + ex.getMessage() + "\n");
|
|
||||||
smxParam.setDistance(-1000);
|
|
||||||
return smxParam;
|
|
||||||
}
|
}
|
||||||
|
sentenceConstituencyParseList.clear();
|
||||||
|
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlist = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, SimpleMatrix> simpleSMXlistVector = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, Integer> sentiment1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, Integer> sentiment2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||||
|
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||||
|
sentiment1.put(sentiment1.size(), RNNCoreAnnotations.getPredictedClass(tree));
|
||||||
|
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||||
|
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||||
|
simpleSMXlist.put(simpleSMXlist.size(), predictions);
|
||||||
|
simpleSMXlistVector.put(simpleSMXlistVector.size() + 1, nodeVector);
|
||||||
|
}
|
||||||
|
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||||
|
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||||
|
sentiment2.put(sentiment2.size() + 1, RNNCoreAnnotations.getPredictedClass(tree));
|
||||||
|
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||||
|
SimpleMatrix nodeVector = RNNCoreAnnotations.getNodeVector(tree);
|
||||||
|
score = simpleSMXlist.values().stream().map((simpleSMX) -> predictions.dot(simpleSMX) * 100).map((dot) -> dot > 50 ? dot - 50 : 50 - dot).map((subtracter) -> {
|
||||||
|
subtracter *= 25;
|
||||||
|
return subtracter;
|
||||||
|
}).map((subtracter) -> subtracter).reduce(score, (accumulator, _item) -> accumulator - _item);
|
||||||
|
for (SimpleMatrix simpleSMX : simpleSMXlistVector.values()) {
|
||||||
|
double dot = nodeVector.dot(simpleSMX);
|
||||||
|
double elementSum = nodeVector.kron(simpleSMX).elementSum();
|
||||||
|
elementSum = Math.round(elementSum * 100.0) / 100.0;
|
||||||
|
if (dot < 0.1) {
|
||||||
|
score += 256;
|
||||||
|
}
|
||||||
|
if (elementSum < 0.1 && elementSum > 0.0) {
|
||||||
|
score += 1300;
|
||||||
|
} else if (elementSum > 0.1 && elementSum < 1.0) {
|
||||||
|
score -= 1100;
|
||||||
|
} else {
|
||||||
|
score -= 1424;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
score -= (sentiment1.size() > sentiment2.size() ? sentiment1.size() - sentiment2.size() : sentiment2.size() - sentiment1.size()) * 500;
|
||||||
|
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
|
||||||
|
List classifyRaw1 = classifier.classifyRaw(str, readerAndWriter);
|
||||||
|
List classifyRaw2 = classifier.classifyRaw(str1, readerAndWriter);
|
||||||
|
score -= (classifyRaw1.size() > classifyRaw2.size() ? classifyRaw1.size() - classifyRaw2.size() : classifyRaw2.size() - classifyRaw1.size()) * 200;
|
||||||
|
int mainSentiment1 = 0;
|
||||||
|
int longest1 = 0;
|
||||||
|
int mainSentiment2 = 0;
|
||||||
|
int longest2 = 0;
|
||||||
|
for (CoreMap sentence : pipelineAnnotation1Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||||
|
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||||
|
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
||||||
|
String partText = sentence.toString();
|
||||||
|
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||||
|
if (partText.length() > longest1) {
|
||||||
|
mainSentiment1 = sentiment;
|
||||||
|
longest1 = partText.length();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (CoreMap sentence : pipelineAnnotation2Sentiment.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||||
|
Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
|
||||||
|
int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
|
||||||
|
SimpleMatrix predictions = RNNCoreAnnotations.getPredictions(tree);
|
||||||
|
String partText = sentence.toString();
|
||||||
|
if (partText.length() > longest2) {
|
||||||
|
mainSentiment2 = sentiment;
|
||||||
|
longest2 = partText.length();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (longest1 != longest2) {
|
||||||
|
long deffLongest = longest1 > longest2 ? longest1 : longest2;
|
||||||
|
long deffshorter = longest1 < longest2 ? longest1 : longest2;
|
||||||
|
if (deffLongest >= (deffshorter * 2) - 1 && deffLongest - deffshorter <= 45) {
|
||||||
|
score += (deffLongest - deffshorter) * 200;
|
||||||
|
} else if (mainSentiment1 != mainSentiment2 && deffLongest - deffshorter > 20 && deffLongest - deffshorter < 45) {
|
||||||
|
score += (deffLongest - deffshorter) * 200;
|
||||||
|
} else {
|
||||||
|
score -= (deffLongest - deffshorter) * 50;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
List<CoreMap> sentences = jmweStrAnnotation1.get(CoreAnnotations.SentencesAnnotation.class);
|
||||||
|
int tokensCounter1 = 0;
|
||||||
|
int tokensCounter2 = 0;
|
||||||
|
int anotatorcounter1 = 0;
|
||||||
|
int anotatorcounter2 = 0;
|
||||||
|
int inflectedCounterPositive1 = 0;
|
||||||
|
int inflectedCounterPositive2 = 0;
|
||||||
|
int inflectedCounterNegative = 0;
|
||||||
|
int MarkedContinuousCounter1 = 0;
|
||||||
|
int MarkedContinuousCounter2 = 0;
|
||||||
|
int UnmarkedPatternCounter = 0;
|
||||||
|
ConcurrentMap<Integer, String> ITokenMapTag1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> ITokenMapTag2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> strTokenStems1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> strTokenStems2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> strTokenForm1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> strTokenForm2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> strTokenGetEntry1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> strTokenGetEntry2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> strTokenGetiPart1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> strTokenGetiPart2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> strTokenEntryPOS1 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
ConcurrentMap<Integer, String> strTokenEntryPOS2 = new MapMaker().concurrencyLevel(2).makeMap();
|
||||||
|
for (CoreMap sentence : sentences) {
|
||||||
|
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
||||||
|
if (token.isInflected()) {
|
||||||
|
inflectedCounterPositive1++;
|
||||||
|
} else {
|
||||||
|
inflectedCounterNegative++;
|
||||||
|
}
|
||||||
|
strTokenForm1.put(strTokenForm1.size() + 1, token.getForm());
|
||||||
|
strTokenGetEntry1.put(strTokenGetEntry1.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
||||||
|
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||||
|
IMWEDesc entry = token.getEntry();
|
||||||
|
MarkedContinuousCounter1 += entry.getMarkedContinuous();
|
||||||
|
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||||
|
for (IMWEDesc.IPart iPart : values) {
|
||||||
|
strTokenGetiPart1.put(strTokenGetiPart1.size() + 1, iPart.getForm());
|
||||||
|
}
|
||||||
|
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
||||||
|
strTokenEntryPOS1.put(strTokenEntryPOS1.size() + 1, strPostPrefix);
|
||||||
|
}
|
||||||
|
for (IToken tokens : token.getTokens()) {
|
||||||
|
ITokenMapTag1.put(ITokenMapTag1.size() + 1, tokens.getTag());
|
||||||
|
for (String strtoken : tokens.getStems()) {
|
||||||
|
strTokenStems1.put(strTokenStems1.size() + 1, strtoken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tokensCounter1++;
|
||||||
|
}
|
||||||
|
anotatorcounter1++;
|
||||||
|
}
|
||||||
|
sentences = jmweStrAnnotation2.get(CoreAnnotations.SentencesAnnotation.class);
|
||||||
|
for (CoreMap sentence : sentences) {
|
||||||
|
for (IMWE<IToken> token : sentence.get(JMWEAnnotation.class)) {
|
||||||
|
if (token.isInflected()) {
|
||||||
|
inflectedCounterPositive2++;
|
||||||
|
} else {
|
||||||
|
inflectedCounterNegative--;
|
||||||
|
}
|
||||||
|
strTokenForm2.put(strTokenForm2.size() + 1, token.getForm());
|
||||||
|
strTokenGetEntry2.put(strTokenGetEntry2.size() + 1, token.getEntry().toString().substring(token.getEntry().toString().length() - 1));
|
||||||
|
Collection<IMWEDesc.IPart> values = token.getPartMap().values();
|
||||||
|
IMWEDesc entry = token.getEntry();
|
||||||
|
MarkedContinuousCounter2 += entry.getMarkedContinuous();
|
||||||
|
UnmarkedPatternCounter += entry.getUnmarkedPattern();
|
||||||
|
for (IMWEDesc.IPart iPart : values) {
|
||||||
|
strTokenGetiPart2.put(strTokenGetiPart2.size() + 1, iPart.getForm());
|
||||||
|
}
|
||||||
|
for (String strPostPrefix : entry.getPOS().getPrefixes()) {
|
||||||
|
strTokenEntryPOS2.put(strTokenEntryPOS2.size() + 1, strPostPrefix);
|
||||||
|
}
|
||||||
|
for (IToken tokens : token.getTokens()) {
|
||||||
|
ITokenMapTag2.put(ITokenMapTag2.size() + 1, tokens.getTag());
|
||||||
|
for (String strtoken : tokens.getStems()) {
|
||||||
|
strTokenStems2.put(strTokenStems2.size() + 1, strtoken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tokensCounter2++;
|
||||||
|
}
|
||||||
|
anotatorcounter2++;
|
||||||
|
}
|
||||||
|
for (String strTokenPos1 : strTokenEntryPOS1.values()) {
|
||||||
|
for (String strTokenPos2 : strTokenEntryPOS2.values()) {
|
||||||
|
if (strTokenPos1.equals(strTokenPos2)) {
|
||||||
|
score += 500;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
score += UnmarkedPatternCounter * 1600;
|
||||||
|
if (MarkedContinuousCounter1 > 0 && MarkedContinuousCounter2 > 0) {
|
||||||
|
score += MarkedContinuousCounter1 > MarkedContinuousCounter2 ? (MarkedContinuousCounter1 - MarkedContinuousCounter2) * 500
|
||||||
|
: (MarkedContinuousCounter2 - MarkedContinuousCounter1) * 500;
|
||||||
|
}
|
||||||
|
for (String strTokeniPart1 : strTokenGetiPart1.values()) {
|
||||||
|
for (String strTokeniPart2 : strTokenGetiPart2.values()) {
|
||||||
|
if (strTokeniPart1.equals(strTokeniPart2)) {
|
||||||
|
score += 400;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (String strTokenEntry1 : strTokenGetEntry1.values()) {
|
||||||
|
for (String strTokenEntry2 : strTokenGetEntry2.values()) {
|
||||||
|
if (strTokenEntry1.equals(strTokenEntry2)) {
|
||||||
|
score += 2500;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (String strmapTag : ITokenMapTag1.values()) {
|
||||||
|
for (String strmapTag1 : ITokenMapTag2.values()) {
|
||||||
|
if (strmapTag.equals(strmapTag1)) {
|
||||||
|
score += 1450;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (String strTokenForm1itr1 : strTokenForm1.values()) {
|
||||||
|
for (String strTokenForm1itr2 : strTokenForm2.values()) {
|
||||||
|
if (strTokenForm1itr1.equals(strTokenForm1itr2)) {
|
||||||
|
score += 2600;
|
||||||
|
} else if (strTokenForm1itr1.contains(strTokenForm1itr2)) {
|
||||||
|
score += 500;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (String strTokenStem : strTokenStems1.values()) {
|
||||||
|
for (String strTokenStem1 : strTokenStems2.values()) {
|
||||||
|
if (strTokenStem.equals(strTokenStem1)) {
|
||||||
|
score += 1500;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (inflectedCounterPositive1 + inflectedCounterPositive2 > inflectedCounterNegative && inflectedCounterNegative > 0) {
|
||||||
|
score += (inflectedCounterPositive1 - inflectedCounterNegative) * 650;
|
||||||
|
}
|
||||||
|
if (inflectedCounterPositive1 > 0 && inflectedCounterPositive2 > 0) {
|
||||||
|
score += ((inflectedCounterPositive1 + inflectedCounterPositive2) - inflectedCounterNegative) * 550;
|
||||||
|
}
|
||||||
|
if (anotatorcounter1 > 1 && anotatorcounter2 > 1) {
|
||||||
|
score += (anotatorcounter1 + anotatorcounter2) * 400;
|
||||||
|
}
|
||||||
|
if (tokensCounter1 > 0 && tokensCounter2 > 0) {
|
||||||
|
score += (tokensCounter1 + tokensCounter2) * 400;
|
||||||
|
} else {
|
||||||
|
score -= tokensCounter1 >= tokensCounter2 ? (tokensCounter1 - tokensCounter2) * 500 : (tokensCounter2 - tokensCounter1) * 500;
|
||||||
|
}
|
||||||
|
LevenshteinDistance leven = new LevenshteinDistance(str, str1);
|
||||||
|
int SentenceScoreDiff = leven.computeLevenshteinDistance();
|
||||||
|
SentenceScoreDiff *= 15;
|
||||||
|
score -= SentenceScoreDiff;
|
||||||
|
System.out.println("Final current score: " + score + "\nSentence 1: " + str + "\nSentence 2: " + str1 + "\n");
|
||||||
|
smxParam.setDistance(score);
|
||||||
return smxParam;
|
return smxParam;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -12,8 +12,6 @@ nohup screen -d -m -S nonroot java -Xmx6800M -jar /home/javatests/ArtificialAut
|
|||||||
screen -ls (number1)
|
screen -ls (number1)
|
||||||
screen -X -S (number1) quit
|
screen -X -S (number1) quit
|
||||||
*/
|
*/
|
||||||
//https://discordapp.com/developers/applications/
|
|
||||||
//https://github.com/Javacord/Javacord
|
|
||||||
package PresentationLayer;
|
package PresentationLayer;
|
||||||
|
|
||||||
import FunctionLayer.CustomError;
|
import FunctionLayer.CustomError;
|
||||||
@ -43,10 +41,12 @@ public class DiscordHandler {
|
|||||||
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
Logger.getLogger(DiscordHandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||||
}
|
}
|
||||||
}).start();
|
}).start();
|
||||||
MYSQLDatahandler.shiftReduceParserInitiate();
|
MYSQLDatahandler.instance.shiftReduceParserInitiate();
|
||||||
MYSQLDatahandler.instance.instantiateExecutor();
|
MYSQLDatahandler.instance.instantiateExecutor();
|
||||||
|
MYSQLDatahandler.instance.updateStringCache();
|
||||||
|
MYSQLDatahandler.instance.instantiateAnnotationMap();
|
||||||
if (MYSQLDatahandler.instance.getstringCacheSize() != 0) {
|
if (MYSQLDatahandler.instance.getstringCacheSize() != 0) {
|
||||||
while (MYSQLDatahandler.instance.getlHMSMXSize() * MYSQLDatahandler.instance.getlHMSMXSize() * 2
|
while (MYSQLDatahandler.instance.getlHMSMXSize() * MYSQLDatahandler.instance.getlHMSMXSize() * 5
|
||||||
< (MYSQLDatahandler.instance.getstringCacheSize()
|
< (MYSQLDatahandler.instance.getstringCacheSize()
|
||||||
* MYSQLDatahandler.instance.getstringCacheSize())
|
* MYSQLDatahandler.instance.getstringCacheSize())
|
||||||
- MYSQLDatahandler.instance.getstringCacheSize()) {
|
- MYSQLDatahandler.instance.getstringCacheSize()) {
|
||||||
@ -62,12 +62,6 @@ public class DiscordHandler {
|
|||||||
strtest = strtest.substring(9, strtest.length() - 1);
|
strtest = strtest.substring(9, strtest.length() - 1);
|
||||||
boolean channelpermissionsDenied = false;
|
boolean channelpermissionsDenied = false;
|
||||||
switch (strtest) {
|
switch (strtest) {
|
||||||
case "Server Area": {
|
|
||||||
if (!event.getServerTextChannel().get().toString().contains("chat-live")) {
|
|
||||||
channelpermissionsDenied = true;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case "Public Area": {
|
case "Public Area": {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
Binary file not shown.
Loading…
Reference in New Issue
Block a user