removed lib files, updated to intellij from netbeans, changed to kotlin coroutines
This commit is contained in:
christian
@@ -1,42 +0,0 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package DataLayer;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import org.apache.commons.dbcp2.BasicDataSource;
|
||||
import DataLayer.settings;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class DBCPDataSourceAutismo {
|
||||
private static BasicDataSource ds = new BasicDataSource();
|
||||
static {
|
||||
try {
|
||||
ds.setDriver(new com.mysql.cj.jdbc.Driver());
|
||||
ds.setUrl(settings.autismo_url);
|
||||
ds.setUsername(settings.autismo_username);
|
||||
ds.setPassword(settings.autismo_password);
|
||||
ds.setMaxTotal(-1);
|
||||
ds.setMinIdle(5);
|
||||
ds.setMaxIdle(-1);
|
||||
ds.setMaxOpenPreparedStatements(100);
|
||||
} catch (SQLException ex) {
|
||||
Logger.getLogger(DBCPDataSourceAutismo.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public static Connection getConnection() throws SQLException {
|
||||
return ds.getConnection();
|
||||
}
|
||||
|
||||
private DBCPDataSourceAutismo() {
|
||||
}
|
||||
}
|
||||
@@ -88,22 +88,6 @@ public class DataMapper {
|
||||
}
|
||||
}
|
||||
|
||||
public static ConcurrentMap<Integer, String> getHLstatsMessages() {
|
||||
ConcurrentMap<Integer, String> hlStatsMessages = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
try (Connection l_cCon = DBCPDataSourceHLstats.getConnection()) {
|
||||
String l_sSQL = "SELECT message FROM `hlstats_Events_Chat`";
|
||||
try (PreparedStatement l_pStatement = l_cCon.prepareStatement(l_sSQL)) {
|
||||
try (ResultSet l_rsSearch = l_pStatement.executeQuery()) {
|
||||
while (l_rsSearch.next()) {
|
||||
hlStatsMessages.put(hlStatsMessages.size() + 1, l_rsSearch.getString(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
Logger.getLogger(DataMapper.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
return hlStatsMessages;
|
||||
}
|
||||
|
||||
public static void CloseConnections(PreparedStatement ps, ResultSet rs, Connection con) {
|
||||
if (rs != null) {
|
||||
|
||||
@@ -1,823 +0,0 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer;
|
||||
|
||||
import DataLayer.DataMapper;
|
||||
import FunctionLayer.StanfordParser.SentimentAnalyzerTest;
|
||||
import FunctionLayer.StanfordParser.SentimentValueCache;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import edu.stanford.nlp.ie.AbstractSequenceClassifier;
|
||||
import edu.stanford.nlp.ie.crf.CRFClassifier;
|
||||
import edu.stanford.nlp.ling.CoreLabel;
|
||||
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
|
||||
import edu.stanford.nlp.pipeline.Annotation;
|
||||
import edu.stanford.nlp.pipeline.CoreDocument;
|
||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
||||
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
|
||||
import edu.stanford.nlp.trees.GrammaticalStructureFactory;
|
||||
import edu.stanford.nlp.trees.TreebankLanguagePack;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import static java.lang.Math.random;
|
||||
import java.net.DatagramPacket;
|
||||
import java.net.DatagramSocket;
|
||||
import java.net.InetAddress;
|
||||
import java.net.SocketException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.ListIterator;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.CompletionService;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorCompletionService;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
import java.util.concurrent.ForkJoinTask;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class Datahandler {
|
||||
|
||||
public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(10, TimeUnit.MINUTES);
|
||||
public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS);
|
||||
public static Datahandler instance = new Datahandler();
|
||||
private static Annotation strAnno;
|
||||
private static Annotation strAnnoSentiment;
|
||||
private static Annotation strAnnoJMWE;
|
||||
private static CoreDocument coreDoc;
|
||||
private static final ConcurrentMap<Integer, String> stringCache = new MapMaker().concurrencyLevel(6).makeMap();
|
||||
private static ConcurrentMap<String, Annotation> pipelineAnnotationCache;
|
||||
private static ConcurrentMap<String, Annotation> pipelineSentimentAnnotationCache;
|
||||
private static ConcurrentMap<String, Annotation> jmweAnnotationCache;
|
||||
private static ConcurrentMap<String, CoreDocument> coreDocumentAnnotationCache;
|
||||
private static ConcurrentMap<String, SentimentValueCache> sentimentCachingMap = new MapMaker().concurrencyLevel(6).makeMap();
|
||||
private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
|
||||
private final Stopwatch stopwatch;
|
||||
private static String similar = "";
|
||||
private static String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz";
|
||||
private static String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz";
|
||||
private static String lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz";
|
||||
private static String taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger";
|
||||
private static String nerModel = "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz";
|
||||
private static String nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz";
|
||||
private static String nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz";
|
||||
private static final String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with";
|
||||
private static MaxentTagger tagger;
|
||||
private static String[] options = {"-maxLength", "100"};
|
||||
private static Properties props = new Properties();
|
||||
private static Properties propsSentiment = new Properties();
|
||||
private static GrammaticalStructureFactory gsf;
|
||||
private static LexicalizedParser lp;
|
||||
private static TreebankLanguagePack tlp;
|
||||
private static AbstractSequenceClassifier<CoreLabel> classifier;
|
||||
// set up Stanford CoreNLP pipeline
|
||||
private static final StanfordCoreNLP pipeline = getPipeLineSetUp();
|
||||
private static StanfordCoreNLP pipelineSentiment;
|
||||
|
||||
public Datahandler() {
|
||||
this.stopwatch = Stopwatch.createUnstarted();
|
||||
this.jmweAnnotationCache = new MapMaker().concurrencyLevel(3).makeMap();
|
||||
this.pipelineAnnotationCache = new MapMaker().concurrencyLevel(4).makeMap();
|
||||
this.pipelineSentimentAnnotationCache = new MapMaker().concurrencyLevel(4).makeMap();
|
||||
this.coreDocumentAnnotationCache = new MapMaker().concurrencyLevel(5).makeMap();
|
||||
}
|
||||
|
||||
public static StanfordCoreNLP getPipeline() {
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
private static StanfordCoreNLP getPipeLineSetUp() {
|
||||
props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse");
|
||||
props.setProperty("parse.model", shiftReduceParserPath);
|
||||
props.setProperty("parse.maxlen", "90");
|
||||
props.setProperty("parse.binaryTrees", "true");
|
||||
props.setProperty("threads", "8");
|
||||
props.setProperty("pos.maxlen", "90");
|
||||
props.setProperty("tokenize.maxlen", "90");
|
||||
props.setProperty("ssplit.maxlen", "90");
|
||||
props.setProperty("lemma.maxlen", "90");
|
||||
props.setProperty("ner.model", nerModel + "," + nerModel2 + "," + nerModel3);
|
||||
props.setProperty("ner.combinationMode", "HIGH_RECALL");
|
||||
props.setProperty("regexner.ignorecase", "true");
|
||||
props.setProperty("ner.fine.regexner.ignorecase", "true");
|
||||
props.setProperty("tokenize.options", "untokenizable=firstDelete");
|
||||
return new StanfordCoreNLP(props);
|
||||
}
|
||||
|
||||
public void shiftReduceParserInitiate() {
|
||||
//got 8 cores
|
||||
CountDownLatch cdl = new CountDownLatch(2);
|
||||
new Thread(() -> {
|
||||
try {
|
||||
classifier = CRFClassifier.getClassifierNoExceptions(nerModel);
|
||||
} catch (ClassCastException ex) {
|
||||
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
cdl.countDown();
|
||||
}).start();
|
||||
new Thread(() -> {
|
||||
propsSentiment.setProperty("parse.model", lexParserEnglishRNN);
|
||||
propsSentiment.setProperty("sentiment.model", sentimentModel);
|
||||
propsSentiment.setProperty("parse.maxlen", "90");
|
||||
propsSentiment.setProperty("threads", "8");
|
||||
propsSentiment.setProperty("pos.maxlen", "90");
|
||||
propsSentiment.setProperty("tokenize.maxlen", "90");
|
||||
propsSentiment.setProperty("ssplit.maxlen", "90");
|
||||
propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword"); //coref too expensive memorywise
|
||||
propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator");
|
||||
propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList);
|
||||
propsSentiment.setProperty("tokenize.options", "untokenizable=firstDelete");
|
||||
pipelineSentiment = new StanfordCoreNLP(propsSentiment);
|
||||
tagger = new MaxentTagger(taggerPath);
|
||||
cdl.countDown();
|
||||
}).start();
|
||||
lp = LexicalizedParser.loadModel(lexParserEnglishRNN, options);
|
||||
tlp = lp.getOp().langpack();
|
||||
gsf = tlp.grammaticalStructureFactory();
|
||||
try {
|
||||
cdl.await();
|
||||
} catch (InterruptedException ex) {
|
||||
//System.out.println("cdl await interrupted: " + ex.getLocalizedMessage() + "\n");
|
||||
}
|
||||
System.out.println("finished shiftReduceParserInitiate\n");
|
||||
}
|
||||
|
||||
public static AbstractSequenceClassifier<CoreLabel> getClassifier() {
|
||||
return classifier;
|
||||
}
|
||||
|
||||
public static void setClassifier(AbstractSequenceClassifier<CoreLabel> classifier) {
|
||||
Datahandler.classifier = classifier;
|
||||
}
|
||||
|
||||
public void updateStringCache() {
|
||||
try {
|
||||
checkIfUpdateStrings();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public static GrammaticalStructureFactory getGsf() {
|
||||
return gsf;
|
||||
}
|
||||
|
||||
public static MaxentTagger getTagger() {
|
||||
return tagger;
|
||||
}
|
||||
|
||||
private Map<Integer, String> getCache() throws SQLException, IOException, CustomError {
|
||||
return DataMapper.getAllStrings();
|
||||
}
|
||||
|
||||
public int getlHMSMXSize() {
|
||||
return lHMSMX.size();
|
||||
}
|
||||
|
||||
public int getstringCacheSize() {
|
||||
return stringCache.size();
|
||||
}
|
||||
|
||||
public void initiateMYSQL() throws SQLException, IOException {
|
||||
try {
|
||||
DataMapper.createTables();
|
||||
stringCache.putAll(getCache());
|
||||
// lHMSMX = DataMapper.getAllRelationScores();
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(Datahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public void addHLstatsMessages() {
|
||||
ConcurrentMap<String, Integer> hlStatsMessages = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<Integer, String> strCacheLocal = stringCache;
|
||||
Collection<String> strs = DataMapper.getHLstatsMessages().values();
|
||||
for (String str : strs) {
|
||||
if (hlStatsMessages.get(str) == null) {
|
||||
hlStatsMessages.put(str, hlStatsMessages.size());
|
||||
}
|
||||
}
|
||||
int capacity = 150;
|
||||
hlStatsMessages.keySet().forEach(str -> {
|
||||
if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) {
|
||||
String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null);
|
||||
if (orElse == null) {
|
||||
MessageResponseHandler.getMessage(str);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void instantiateAnnotationMapJMWE() {
|
||||
if (!stringCache.isEmpty()) {
|
||||
ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(stringCache.values());
|
||||
for (Entry<String, Annotation> entries : jmweAnnotation.entrySet()) {
|
||||
jmweAnnotationCache.put(entries.getKey(), entries.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void instantiateAnnotationMap() {
|
||||
if (!stringCache.isEmpty()) {
|
||||
ConcurrentMap<String, Annotation> Annotationspipeline = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
ConcurrentMap<String, Annotation> AnnotationspipelineSentiment = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
stringCache.values().parallelStream().forEach(str -> {
|
||||
Annotation strAnno = new Annotation(str);
|
||||
strAnno.compact();
|
||||
Annotationspipeline.put(str, strAnno);
|
||||
Annotation strAnno2 = new Annotation(str);
|
||||
strAnno2.compact();
|
||||
AnnotationspipelineSentiment.put(str, strAnno2);
|
||||
});
|
||||
ConcurrentMap<String, CoreDocument> coreDocumentpipelineMap = getMultipleCoreDocumentsWaySuggestion(stringCache.values(), pipeline);
|
||||
pipeline.annotate(Annotationspipeline.values());
|
||||
pipelineSentiment.annotate(AnnotationspipelineSentiment.values());
|
||||
Annotationspipeline.entrySet().forEach(pipelineEntry -> {
|
||||
//relatively experimental change
|
||||
pipelineEntry.getValue().compact();
|
||||
pipelineAnnotationCache.put(pipelineEntry.getKey(), pipelineEntry.getValue());
|
||||
});
|
||||
AnnotationspipelineSentiment.entrySet().forEach(pipelineEntry -> {
|
||||
pipelineEntry.getValue().compact();
|
||||
pipelineSentimentAnnotationCache.put(pipelineEntry.getKey(), pipelineEntry.getValue());
|
||||
});
|
||||
coreDocumentpipelineMap.entrySet().stream().forEach(CD -> {
|
||||
coreDocumentAnnotationCache.put(CD.getKey(), CD.getValue());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private ConcurrentMap<Integer, String> futuresReturnOverallEvaluation(List<SimilarityMatrix> similarityMatrixes) {
|
||||
ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(6).makeMap();
|
||||
if (!similarityMatrixes.isEmpty()) {
|
||||
int iterator = 0;
|
||||
for (SimilarityMatrix SMX : similarityMatrixes) {
|
||||
final Double scoreRelationNewMsgToRecentMsg = SMX.getDistance();
|
||||
if (scoreRelationNewMsgToRecentMsg > 0.0) {
|
||||
strmapreturn = addSMXToMapReturn(strmapreturn, SMX);
|
||||
}
|
||||
//System.out.println("scoreRelationNewMsgToRecentMsg: " + scoreRelationNewMsgToRecentMsg + "\niterator: " + iterator);
|
||||
iterator++;
|
||||
}
|
||||
}
|
||||
return strmapreturn;
|
||||
}
|
||||
|
||||
private ConcurrentMap<Integer, String> addSMXToMapReturn(ConcurrentMap<Integer, String> strmapreturn, SimilarityMatrix SMX) {
|
||||
if (!strmapreturn.containsValue(SMX.getPrimaryString())) {
|
||||
strmapreturn.put(strmapreturn.size(), SMX.getPrimaryString());
|
||||
String transmittedStr = SMX.getSecondaryString();
|
||||
SentimentValueCache cacheValue1 = SMX.getCacheValue1();
|
||||
SentimentValueCache cacheValue2 = SMX.getCacheValue2();
|
||||
if (cacheValue1 != null && !sentimentCachingMap.keySet().contains(SMX.getPrimaryString())) {
|
||||
sentimentCachingMap.put(SMX.getSecondaryString(), SMX.getCacheValue1());
|
||||
}
|
||||
if (cacheValue2 != null && !sentimentCachingMap.keySet().contains(transmittedStr)) {
|
||||
sentimentCachingMap.put(transmittedStr, SMX.getCacheValue2());
|
||||
}
|
||||
}
|
||||
return strmapreturn;
|
||||
}
|
||||
|
||||
private List<SimilarityMatrix> StrComparringNoSentenceRelationMap(
|
||||
ConcurrentMap<Integer, String> strCacheLocal, Collection<String> strCollection, ConcurrentMap<String, Annotation> localJMWEMap,
|
||||
ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
|
||||
ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
|
||||
ExecutorService threadPool = Executors.newCachedThreadPool();
|
||||
CompletionService<SimilarityMatrix> ecs = new ExecutorCompletionService<>(threadPool);
|
||||
int index = 0;
|
||||
int prefix_size = 150;
|
||||
SentimentValueCache sentimentCacheStr = sentimentCachingMap.getOrDefault(strCollection, null);
|
||||
List<SimilarityMatrix> smxReturnList = new ArrayList();
|
||||
if (strCacheLocal.size() < prefix_size)
|
||||
{
|
||||
for (String colStr : strCollection)
|
||||
{
|
||||
strCacheLocal.put(strCacheLocal.size(), colStr);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
for (String str1 : strCollection) {
|
||||
for (String str : strCollection) {
|
||||
if (!str.equals(str1)) {
|
||||
SimilarityMatrix SMXInit = new SimilarityMatrix(str, str1);
|
||||
SentimentValueCache sentimentCacheStr1 = sentimentCachingMap.getOrDefault(str1, null);
|
||||
Callable<SimilarityMatrix> worker;
|
||||
if (stringCache.size() < prefix_size) {
|
||||
worker = new SentimentAnalyzerTest(str, str1, SMXInit,
|
||||
localJMWEMap.get(str), localJMWEMap.get(str1), localPipelineAnnotation.get(str),
|
||||
localPipelineAnnotation.get(str1), localPipelineSentimentAnnotation.get(str),
|
||||
localPipelineSentimentAnnotation.get(str1), localCoreDocumentMap.get(str), localCoreDocumentMap.get(str1), sentimentCacheStr, sentimentCacheStr1);
|
||||
} else {
|
||||
worker = new SentimentAnalyzerTest(str, str1, SMXInit,
|
||||
localJMWEMap.get(str), jmweAnnotationCache.get(str1), localPipelineAnnotation.get(str),
|
||||
pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str),
|
||||
pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1), sentimentCacheStr, sentimentCacheStr1);
|
||||
}
|
||||
ecs.submit(worker);
|
||||
index++;
|
||||
if (index % 1000 == 0 && index > 0) {
|
||||
for (int i = 0; i < index; i++) {
|
||||
try {
|
||||
Future<SimilarityMatrix> take = ecs.take();
|
||||
SimilarityMatrix smx = take.get();
|
||||
if (smx != null) {
|
||||
smxReturnList.add(smx);
|
||||
}
|
||||
} catch (InterruptedException | ExecutionException ex) {
|
||||
//
|
||||
}
|
||||
}
|
||||
index = 0;
|
||||
//System.out.println("smxReturnList size iterating ECS.take(): " + smxReturnList.size());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < index; i++) {
|
||||
try {
|
||||
Future<SimilarityMatrix> take = ecs.take();
|
||||
SimilarityMatrix smx = take.get();
|
||||
double distance_requirement = 1500.0;
|
||||
if (smx != null && smx.getDistance() > distance_requirement) {
|
||||
smxReturnList.add(smx);
|
||||
}
|
||||
} catch (InterruptedException | ExecutionException ex) {
|
||||
//
|
||||
}
|
||||
}
|
||||
//System.out.println("smxReturnList size: " + smxReturnList.size());
|
||||
threadPool.shutdown();
|
||||
return smxReturnList;
|
||||
}
|
||||
|
||||
private ConcurrentMap<Integer, String> stringIteratorComparator(ConcurrentMap<Integer, String> strmap,
|
||||
ConcurrentMap<Integer, String> strCacheLocal, ConcurrentMap<String, Annotation> localJMWEMap,
|
||||
ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
|
||||
ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
|
||||
//System.out.println("strmap siuze: " + strmap.size());
|
||||
List<SimilarityMatrix> StrComparringNoSentenceRelationMap = StrComparringNoSentenceRelationMap(strCacheLocal, strmap.values(),
|
||||
localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap);
|
||||
Collections.sort(StrComparringNoSentenceRelationMap, (e1, e2) -> e1.getPrimaryString().compareTo(e2.getPrimaryString()));
|
||||
ConcurrentMap<Integer, String> strmapreturn = futuresReturnOverallEvaluation(StrComparringNoSentenceRelationMap);
|
||||
//System.out.println("strmapreturn size: " + strmapreturn.size());
|
||||
return strmapreturn;
|
||||
}
|
||||
|
||||
private ConcurrentMap<Integer, String> removeNonSensicalStrings(ConcurrentMap<Integer, String> strmap) {
|
||||
final ConcurrentMap<Integer, String> strCacheLocal = stringCache;
|
||||
final ConcurrentMap<String, Annotation> localJMWEMap = getMultipleJMWEAnnotation(strmap.values());
|
||||
final ConcurrentMap<String, Annotation> localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values());
|
||||
final ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation = getMultiplePipelineSentimentAnnotation(strmap.values());
|
||||
final ConcurrentMap<String, CoreDocument> localCoreDocumentMap = getMultipleCoreDocumentsWaySuggestion(strmap.values(), pipeline);
|
||||
return stringIteratorComparator(strmap, strCacheLocal, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap);
|
||||
}
|
||||
|
||||
public synchronized void checkIfUpdateStrings() throws CustomError {
|
||||
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
|
||||
ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
|
||||
System.out.println("str size: " + str.size());
|
||||
str = filterContent(str);
|
||||
str = removeNonSensicalStrings(str);
|
||||
//System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n");
|
||||
str = annotationCacheUpdate(str);
|
||||
System.out.println("annotationCacheUpdate str size POST: " + str.size() + "\n");
|
||||
ConcurrentMap<Integer, String> strf = str;
|
||||
if (!stringCache.isEmpty()) {
|
||||
new Thread(() -> {
|
||||
try {
|
||||
DataMapper.InsertMYSQLStrings(strf);
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(Datahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
MessageResponseHandler.setStr(new MapMaker().concurrencyLevel(6).makeMap());
|
||||
}).start();
|
||||
} else {
|
||||
try {
|
||||
DataMapper.InsertMYSQLStrings(strf);
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(Datahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
MessageResponseHandler.setStr(new MapMaker().concurrencyLevel(2).makeMap());
|
||||
}
|
||||
if (!stopwatch.isRunning()) {
|
||||
stopwatch.start();
|
||||
} else {
|
||||
stopwatch.reset();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String trimString(String str) {
|
||||
str = str.trim();
|
||||
if (str.startsWith("<@")) {
|
||||
str = str.substring(str.indexOf("> ") + 2);
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
private String getResponseFutures(String strF) {
|
||||
List<String> values_copy = new ArrayList<String>(stringCache.values());
|
||||
Collections.shuffle(values_copy);
|
||||
double preRelationUserCounters = -155000.0;
|
||||
List<String> concurrentRelations = new ArrayList();
|
||||
for (String str1 : values_copy) {
|
||||
if (!strF.equals(str1)) {
|
||||
SentimentValueCache sentimentCacheStr1 = sentimentCachingMap.getOrDefault(str1, null);
|
||||
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(strF, str1, new SimilarityMatrix(strF, str1),
|
||||
strAnnoJMWE, jmweAnnotationCache.get(str1), strAnno,
|
||||
pipelineAnnotationCache.get(str1), strAnnoSentiment,
|
||||
pipelineSentimentAnnotationCache.get(str1), coreDoc, coreDocumentAnnotationCache.get(str1), null, sentimentCacheStr1);
|
||||
try {
|
||||
SimilarityMatrix getSMX = worker.call();
|
||||
if (getSMX != null) {
|
||||
Double scoreRelationLastUserMsg = getSMX.getDistance();
|
||||
if (scoreRelationLastUserMsg > preRelationUserCounters) {
|
||||
preRelationUserCounters = scoreRelationLastUserMsg;
|
||||
concurrentRelations.add(getSMX.getSecondaryString());
|
||||
//System.out.println("secondary: " + getSMX.getSecondaryString() + "\nDistance: " + getSMX.getDistance() + "\n");
|
||||
//System.out.println("SUCESS concurrentRelationsMap size: " + concurrentRelations.size() + "\n");
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
StringBuilder SB = new StringBuilder();
|
||||
double randomLenghtPermit = strF.length() * ((Math.random() * Math.random() * Math.random()) * 5);
|
||||
Collections.reverse(concurrentRelations);
|
||||
if (concurrentRelations.isEmpty()) {
|
||||
return "failure, preventing stuckness";
|
||||
}
|
||||
String firstRelation = concurrentRelations.get(0);
|
||||
for (String secondaryRelation : concurrentRelations) {
|
||||
if (SB.toString().length() > randomLenghtPermit && !SB.toString().isEmpty()) {
|
||||
break;
|
||||
}
|
||||
boolean append = appendToString(firstRelation, secondaryRelation);
|
||||
if (append) {
|
||||
SB.append(secondaryRelation).append(" ");
|
||||
}
|
||||
}
|
||||
return SB.toString();
|
||||
}
|
||||
|
||||
private boolean appendToString(String firstRelation, String secondaryRelation) {
|
||||
if (firstRelation.equals(secondaryRelation)) {
|
||||
return true;
|
||||
}
|
||||
Double scoreRelationStrF = getScoreRelationStrF(firstRelation, secondaryRelation);
|
||||
if (scoreRelationStrF > 1900) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public String getResponseMsg(String str) throws CustomError {
|
||||
String strF = trimString(str);
|
||||
getSingularAnnotation(strF);
|
||||
return getResponseFutures(strF);
|
||||
}
|
||||
|
||||
public void getSingularAnnotation(String str) {
|
||||
strAnno = new Annotation(str);
|
||||
strAnno.compact();
|
||||
pipeline.annotate(strAnno);
|
||||
strAnnoSentiment = new Annotation(str);
|
||||
strAnnoSentiment.compact();
|
||||
pipelineSentiment.annotate(strAnnoSentiment);
|
||||
List<String> notactualList = new ArrayList();
|
||||
notactualList.add(str);
|
||||
ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(notactualList);
|
||||
strAnnoJMWE = jmweAnnotation.values().iterator().next();
|
||||
strAnnoJMWE.compact();
|
||||
CoreDocument coreDocument = new CoreDocument(str);
|
||||
pipeline.annotate(coreDocument);
|
||||
coreDoc = coreDocument;
|
||||
}
|
||||
|
||||
private static ConcurrentMap<String, Annotation> getMultipleJMWEAnnotation(Collection<String> str) {
|
||||
ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str);
|
||||
return jmweAnnotation;
|
||||
}
|
||||
|
||||
private static ConcurrentMap<String, Annotation> getMultiplePipelineAnnotation(Collection<String> str) {
|
||||
ConcurrentMap<String, Annotation> pipelineAnnotationMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String str1 : str) {
|
||||
Annotation strAnno1 = new Annotation(str1);
|
||||
pipelineAnnotationMap.put(str1, strAnno1);
|
||||
}
|
||||
pipeline.annotate(pipelineAnnotationMap.values());
|
||||
return pipelineAnnotationMap;
|
||||
}
|
||||
|
||||
private static ConcurrentMap<String, Annotation> getMultiplePipelineSentimentAnnotation(Collection<String> str) {
|
||||
ConcurrentMap<String, Annotation> pipelineAnnotationMap = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
for (String str1 : str) {
|
||||
Annotation strAnno1 = new Annotation(str1);
|
||||
pipelineAnnotationMap.put(str1, strAnno1);
|
||||
}
|
||||
pipelineSentiment.annotate(pipelineAnnotationMap.values());
|
||||
return pipelineAnnotationMap;
|
||||
}
|
||||
|
||||
private Double getScoreRelationNewMsgToRecentMsg(String str, String mostRecentMsg) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(str, mostRecentMsg);
|
||||
SentimentValueCache cacheSentiment1 = sentimentCachingMap.getOrDefault(str, null);
|
||||
SentimentValueCache cacheSentiment2 = sentimentCachingMap.getOrDefault(mostRecentMsg, null);
|
||||
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, mostRecentMsg, SMX,
|
||||
jmweAnnotationCache.get(str), jmweAnnotationCache.get(mostRecentMsg), pipelineAnnotationCache.get(str),
|
||||
pipelineAnnotationCache.get(mostRecentMsg), pipelineSentimentAnnotationCache.get(str),
|
||||
pipelineSentimentAnnotationCache.get(mostRecentMsg), coreDocumentAnnotationCache.get(str),
|
||||
coreDocumentAnnotationCache.get(mostRecentMsg), cacheSentiment1, cacheSentiment2);
|
||||
SimilarityMatrix callSMX = null;
|
||||
try {
|
||||
callSMX = worker.call();
|
||||
} catch (Exception ex) {
|
||||
Logger.getLogger(Datahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
if (callSMX != null) {
|
||||
double smxDistance = callSMX.getDistance();
|
||||
return smxDistance;
|
||||
}
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
private Double getScoreRelationStrF(String str, String mostRecentMsg) {
|
||||
SimilarityMatrix SMX = new SimilarityMatrix(str, mostRecentMsg);
|
||||
SentimentValueCache cacheSentiment1 = sentimentCachingMap.getOrDefault(str, null);
|
||||
SentimentValueCache cacheSentiment2 = sentimentCachingMap.getOrDefault(mostRecentMsg, null);
|
||||
Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, mostRecentMsg, SMX,
|
||||
strAnnoJMWE, jmweAnnotationCache.get(mostRecentMsg), strAnno,
|
||||
pipelineAnnotationCache.get(mostRecentMsg), strAnnoSentiment,
|
||||
pipelineSentimentAnnotationCache.get(mostRecentMsg), coreDoc, coreDocumentAnnotationCache.get(mostRecentMsg), cacheSentiment1, cacheSentiment2);
|
||||
SimilarityMatrix callSMX = null;
|
||||
try {
|
||||
callSMX = worker.call();
|
||||
} catch (Exception ex) {
|
||||
Logger.getLogger(Datahandler.class
|
||||
.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
if (callSMX != null) {
|
||||
double smxDistance = callSMX.getDistance();
|
||||
return smxDistance;
|
||||
}
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) {
|
||||
ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap();
|
||||
str.values().forEach(str1 -> {
|
||||
if (!str1.isEmpty() && str1.length() > 3) {
|
||||
str1 = str1.trim();
|
||||
if (str1.contains("PM*")) {
|
||||
str1 = str1.substring(str1.indexOf("PM*") + 3);
|
||||
}
|
||||
if (str1.contains("AM*")) {
|
||||
str1 = str1.substring(str1.indexOf("AM*") + 3);
|
||||
}
|
||||
if (str1.contains("?") || str1.contains("°"))
|
||||
{
|
||||
if (!str1.contains("http"))
|
||||
{
|
||||
str1 = str1.replace("?", " <:wlenny:514861023002624001> ");
|
||||
str1 = str1.replace("°", " <:wlenny:514861023002624001> ");
|
||||
}
|
||||
}
|
||||
|
||||
if (str1.contains("(Counter-Terrorist)")) {
|
||||
str1 = str1.replace("(Counter-Terrorist)", " ");
|
||||
}
|
||||
if (str1.contains("(Terrorist)")) {
|
||||
str1 = str1.replace("(Terrorist)", " ");
|
||||
}
|
||||
if (str1.contains("(Spectator)")) {
|
||||
str1 = str1.replace("(Spectator)", " ");
|
||||
}
|
||||
if (str1.contains("*DEAD*")) {
|
||||
str1 = str1.replace("*DEAD*", " ");
|
||||
}
|
||||
if (str1.contains("{red}")) {
|
||||
str1 = str1.replace("{red}", " ");
|
||||
}
|
||||
if (str1.contains("{orange}")) {
|
||||
str1 = str1.replace("{orange}", " ");
|
||||
}
|
||||
if (str1.contains("{yellow}")) {
|
||||
str1 = str1.replace("{yellow}", " ");
|
||||
}
|
||||
if (str1.contains("{green}")) {
|
||||
str1 = str1.replace("{green}", " ");
|
||||
}
|
||||
if (str1.contains("{lightblue}")) {
|
||||
str1 = str1.replace("{lightblue}", " ");
|
||||
}
|
||||
if (str1.contains("{blue}")) {
|
||||
str1 = str1.replace("{blue}", " ");
|
||||
}
|
||||
if (str1.contains("{purple}")) {
|
||||
str1 = str1.replace("{purple}", " ");
|
||||
}
|
||||
if (str1.contains("{white}")) {
|
||||
str1 = str1.replace("{white}", " ");
|
||||
}
|
||||
if (str1.contains("{fullblue}")) {
|
||||
str1 = str1.replace("{fullblue}", " ");
|
||||
}
|
||||
if (str1.contains("{cyan}")) {
|
||||
str1 = str1.replace("{cyan}", " ");
|
||||
}
|
||||
if (str1.contains("{lime}")) {
|
||||
str1 = str1.replace("{lime}", " ");
|
||||
}
|
||||
if (str1.contains("{deeppink}")) {
|
||||
str1 = str1.replace("{deeppink}", " ");
|
||||
}
|
||||
if (str1.contains("{slategray}")) {
|
||||
str1 = str1.replace("{slategray}", " ");
|
||||
}
|
||||
if (str1.contains("{dodgerblue}")) {
|
||||
str1 = str1.replace("{dodgerblue}", " ");
|
||||
}
|
||||
if (str1.contains("{black}")) {
|
||||
str1 = str1.replace("{black}", " ");
|
||||
}
|
||||
if (str1.contains("{orangered}")) {
|
||||
str1 = str1.replace("{orangered}", " ");
|
||||
}
|
||||
if (str1.contains("{darkorchid}")) {
|
||||
str1 = str1.replace("{darkorchid}", " ");
|
||||
}
|
||||
if (str1.contains("{pink}")) {
|
||||
str1 = str1.replace("{pink}", " ");
|
||||
}
|
||||
if (str1.contains("{lightyellow}")) {
|
||||
str1 = str1.replace("{lightyellow}", " ");
|
||||
}
|
||||
if (str1.contains("{chocolate}")) {
|
||||
str1 = str1.replace("{chocolate}", " ");
|
||||
}
|
||||
if (str1.contains("{beige}")) {
|
||||
str1 = str1.replace("{beige}", " ");
|
||||
}
|
||||
if (str1.contains("{azure}")) {
|
||||
str1 = str1.replace("{azure}", " ");
|
||||
}
|
||||
if (str1.contains("{yellowgreen}")) {
|
||||
str1 = str1.replace("{yellowgreen}", " ");
|
||||
}
|
||||
str1 = str1.trim();
|
||||
if (str1.length() > 2 && (!str1.startsWith("!"))) {
|
||||
strlistreturn.put(strlistreturn.size(), str1);
|
||||
}
|
||||
}
|
||||
});
|
||||
return strlistreturn;
|
||||
}
|
||||
|
||||
private ConcurrentMap<Integer, String> annotationCacheUpdate(ConcurrentMap<Integer, String> strmap) {
|
||||
ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strmap.values());
|
||||
for (Entry<String, Annotation> jmweitr : jmweAnnotation.entrySet()) {
|
||||
jmweAnnotationCache.put(jmweitr.getKey(), jmweitr.getValue());
|
||||
}
|
||||
ConcurrentMap<String, Annotation> Annotationspipeline = new MapMaker().concurrencyLevel(4).makeMap();
|
||||
ConcurrentMap<String, Annotation> AnnotationspipelineSentiment = new MapMaker().concurrencyLevel(4).makeMap();
|
||||
ConcurrentMap<String, CoreDocument> coreDocumentpipelineMap = getMultipleCoreDocumentsWaySuggestion(strmap.values(), pipeline);
|
||||
strmap.values().forEach(str -> {
|
||||
Annotation strAnno1 = new Annotation(str);
|
||||
Annotationspipeline.put(str, strAnno1);
|
||||
Annotation strAnno2 = new Annotation(str);
|
||||
AnnotationspipelineSentiment.put(str, strAnno2);
|
||||
stringCache.put(stringCache.size() + 1, str);
|
||||
});
|
||||
pipeline.annotate(Annotationspipeline.values());
|
||||
pipelineSentiment.annotate(AnnotationspipelineSentiment.values());
|
||||
Annotationspipeline.entrySet().forEach(pipelineEntry -> {
|
||||
if (pipelineEntry != null) {
|
||||
pipelineAnnotationCache.put(pipelineEntry.getKey(), pipelineEntry.getValue());
|
||||
}
|
||||
});
|
||||
AnnotationspipelineSentiment.entrySet().forEach(pipelineEntry -> {
|
||||
if (pipelineEntry != null) {
|
||||
pipelineSentimentAnnotationCache.put(pipelineEntry.getKey(), pipelineEntry.getValue());
|
||||
}
|
||||
});
|
||||
coreDocumentpipelineMap.entrySet().forEach(coreDocumentEntry -> {
|
||||
coreDocumentAnnotationCache.put(coreDocumentEntry.getKey(), coreDocumentEntry.getValue());
|
||||
});
|
||||
return strmap;
|
||||
}
|
||||
|
||||
public int getMessageOverHead() {
|
||||
return stringCache.values().size() - (stringCache.values().size() / 10);
|
||||
}
|
||||
|
||||
public void update_autismo_socket_msg() {
|
||||
try {
|
||||
try (DatagramSocket serverSocket = new DatagramSocket(48477)) {
|
||||
try (DatagramSocket serverSocket1 = new DatagramSocket(48478)) {
|
||||
byte[] receiveData = new byte[4096];
|
||||
InetAddress IPAddress = InetAddress.getByName("144.76.218.19");
|
||||
DatagramPacket receivePacket = new DatagramPacket(receiveData, receiveData.length);
|
||||
while (true) {
|
||||
serverSocket.receive(receivePacket);
|
||||
String sentence = new String(receivePacket.getData(), 0, receivePacket.getLength());
|
||||
sentence = sentence.replace("clientmessage:", "");
|
||||
String getResponseMsg = getResponseMsg(sentence);
|
||||
byte[] sendData = getResponseMsg.getBytes("UTF-8");
|
||||
DatagramPacket sendPacket = new DatagramPacket(sendData, sendData.length, IPAddress, 48477);
|
||||
serverSocket.send(sendPacket);
|
||||
|
||||
receivePacket = new DatagramPacket(receiveData, receiveData.length);
|
||||
serverSocket1.receive(receivePacket);
|
||||
sentence = new String(receivePacket.getData(), 0, receivePacket.getLength());
|
||||
sentence = sentence.replace("clientmessage:", "");
|
||||
getResponseMsg = getResponseMsg(sentence);
|
||||
sendData = getResponseMsg.getBytes("UTF-8");
|
||||
sendPacket = new DatagramPacket(sendData, sendData.length, IPAddress, 48478);
|
||||
serverSocket1.send(sendPacket);
|
||||
}
|
||||
}
|
||||
} catch (CustomError ex) {
|
||||
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
} catch (SocketException ex) {
|
||||
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
} catch (UnsupportedEncodingException ex) {
|
||||
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
} catch (IOException ex) {
|
||||
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static class AnnotationCollector<T> implements Consumer<T> {
|
||||
|
||||
private static int i = 0;
|
||||
private List<T> annotationsT = new ArrayList();
|
||||
|
||||
@Override
|
||||
public void accept(T ann) {
|
||||
//System.out.println("adding ann: " + ann.toString());
|
||||
annotationsT.add(ann);
|
||||
}
|
||||
}
|
||||
|
||||
public static ConcurrentMap<String, CoreDocument> getMultipleCoreDocumentsWaySuggestion(Collection<String> str, StanfordCoreNLP localNLP) {
|
||||
AnnotationCollector<Annotation> annCollector = new AnnotationCollector();
|
||||
for (String exampleString : str) {
|
||||
localNLP.annotate(new Annotation(exampleString), annCollector);
|
||||
annCollector.i++;
|
||||
//System.out.println("iterator: " + annCollector.i + "\nstr size: " + str.size() + "\n");
|
||||
}
|
||||
try {
|
||||
Thread.sleep(8000);
|
||||
} catch (InterruptedException ex) {
|
||||
Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
ConcurrentMap<String, CoreDocument> annotationreturnMap = new MapMaker().concurrencyLevel(6).makeMap();
|
||||
for (Annotation ann : annCollector.annotationsT) {
|
||||
if (ann != null) {
|
||||
ann.compact();
|
||||
CoreDocument CD = new CoreDocument(ann);
|
||||
annotationreturnMap.put(CD.text(), CD);
|
||||
//System.out.println("CD text:" + CD.text() + "\niterator: " + iterator + "\nsize: " + annCollector.annotationsT.size());
|
||||
}
|
||||
}
|
||||
return annotationreturnMap;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,804 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer
|
||||
|
||||
import DataLayer.DataMapper
|
||||
import FunctionLayer.StanfordParser.SentimentAnalyzerTest
|
||||
import FunctionLayer.StanfordParser.SentimentValueCache
|
||||
import com.google.common.base.Stopwatch
|
||||
import com.google.common.collect.MapMaker
|
||||
import edu.stanford.nlp.ie.AbstractSequenceClassifier
|
||||
import edu.stanford.nlp.ie.crf.CRFClassifier
|
||||
import edu.stanford.nlp.ling.CoreLabel
|
||||
import edu.stanford.nlp.parser.lexparser.LexicalizedParser
|
||||
import edu.stanford.nlp.pipeline.Annotation
|
||||
import edu.stanford.nlp.pipeline.CoreDocument
|
||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP
|
||||
import edu.stanford.nlp.tagger.maxent.MaxentTagger
|
||||
import edu.stanford.nlp.trees.GrammaticalStructureFactory
|
||||
import edu.stanford.nlp.trees.TreebankLanguagePack
|
||||
import kotlinx.coroutines.*
|
||||
import java.io.IOException
|
||||
import java.io.UnsupportedEncodingException
|
||||
import java.lang.Runnable
|
||||
import java.net.DatagramPacket
|
||||
import java.net.DatagramSocket
|
||||
import java.net.InetAddress
|
||||
import java.net.SocketException
|
||||
import java.sql.SQLException
|
||||
import java.util.*
|
||||
import java.util.concurrent.*
|
||||
import java.util.function.Consumer
|
||||
import java.util.logging.Level
|
||||
import java.util.logging.Logger
|
||||
import kotlin.collections.ArrayList
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
class Datahandler {
|
||||
private val lHMSMX: LinkedHashMap<Any?, Any?> = LinkedHashMap<Any?, Any?>()
|
||||
private val stopwatch: Stopwatch
|
||||
fun shiftReduceParserInitiate() {
|
||||
val cdl = CountDownLatch(2)
|
||||
Thread(Runnable {
|
||||
try {
|
||||
classifier = CRFClassifier.getClassifierNoExceptions(nerModel)
|
||||
} catch (ex: ClassCastException) {
|
||||
Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
cdl.countDown()
|
||||
}).start()
|
||||
Thread(Runnable {
|
||||
propsSentiment.setProperty("parse.model", lexParserEnglishRNN)
|
||||
propsSentiment.setProperty("sentiment.model", sentimentModel)
|
||||
propsSentiment.setProperty("parse.maxlen", "90")
|
||||
propsSentiment.setProperty("threads", "5")
|
||||
propsSentiment.setProperty("pos.maxlen", "90")
|
||||
propsSentiment.setProperty("tokenize.maxlen", "90")
|
||||
propsSentiment.setProperty("ssplit.maxlen", "90")
|
||||
propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword") //coref too expensive memorywise
|
||||
propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator")
|
||||
propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList)
|
||||
propsSentiment.setProperty("tokenize.options", "untokenizable=firstKeep")
|
||||
pipelineSentiment = StanfordCoreNLP(propsSentiment)
|
||||
tagger = MaxentTagger(taggerPath)
|
||||
cdl.countDown()
|
||||
}).start()
|
||||
lp = LexicalizedParser.loadModel(lexParserEnglishRNN, *options)
|
||||
|
||||
tlp = lp.getOp().langpack()
|
||||
gsf = tlp.grammaticalStructureFactory()
|
||||
try {
|
||||
cdl.await()
|
||||
} catch (ex: InterruptedException) {
|
||||
//System.out.println("cdl await interrupted: " + ex.getLocalizedMessage() + "\n");
|
||||
}
|
||||
println("finished shiftReduceParserInitiate\n")
|
||||
}
|
||||
|
||||
fun updateStringCache() {
|
||||
try {
|
||||
checkIfUpdateStrings()
|
||||
} catch (ex: CustomError) {
|
||||
Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
}
|
||||
|
||||
@get:Throws(SQLException::class, IOException::class, CustomError::class)
|
||||
private val cache: Map<Int, String?>
|
||||
private get() = DataMapper.getAllStrings()
|
||||
|
||||
@Throws(SQLException::class, IOException::class)
|
||||
fun initiateMYSQL() {
|
||||
try {
|
||||
DataMapper.createTables()
|
||||
stringCache.putAll(cache)
|
||||
// lHMSMX = DataMapper.getAllRelationScores();
|
||||
} catch (ex: CustomError) {
|
||||
Logger.getLogger(Datahandler::class.java
|
||||
.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
}
|
||||
|
||||
fun instantiateAnnotationMapJMWE() {
|
||||
if (!stringCache.isEmpty()) {
|
||||
val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(stringCache.values)
|
||||
for ((key, value) in jmweAnnotation) {
|
||||
jmweAnnotationCache[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun instantiateAnnotationMap() {
|
||||
if (!stringCache.isEmpty()) {
|
||||
val Annotationspipeline = MapMaker().concurrencyLevel(4).makeMap<String?, Annotation>()
|
||||
val AnnotationspipelineSentiment = MapMaker().concurrencyLevel(4).makeMap<String?, Annotation>()
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (str in stringCache.values) {
|
||||
val strAnno = Annotation(str)
|
||||
strAnno.compact()
|
||||
Annotationspipeline[str] = strAnno
|
||||
val strAnno2 = Annotation(str)
|
||||
strAnno2.compact()
|
||||
AnnotationspipelineSentiment[str] = strAnno2
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join();
|
||||
}
|
||||
val coreDocumentpipelineMap = getMultipleCoreDocumentsWaySuggestion(stringCache.values, pipeline)
|
||||
pipeline.annotate(Annotationspipeline.values)
|
||||
pipelineSentiment!!.annotate(AnnotationspipelineSentiment.values)
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (i in Annotationspipeline.entries) {
|
||||
i.value.compact()
|
||||
pipelineAnnotationCache[i.key] = i.value
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (i in AnnotationspipelineSentiment.entries) {
|
||||
i.value.compact()
|
||||
pipelineSentimentAnnotationCache[i.key] = i.value
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (i in coreDocumentpipelineMap.entries) {
|
||||
coreDocumentAnnotationCache[i.key] = i.value
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun futuresReturnOverallEvaluation(similarityMatrixes: List<SimilarityMatrix?>): ConcurrentMap<Int?, String?> {
|
||||
var strmapreturn = MapMaker().concurrencyLevel(6).makeMap<Int?, String?>()
|
||||
if (!similarityMatrixes.isEmpty()) {
|
||||
var iterator = 0
|
||||
for (SMX in similarityMatrixes) {
|
||||
val scoreRelationNewMsgToRecentMsg = SMX!!.distance
|
||||
if (scoreRelationNewMsgToRecentMsg > 0.0) {
|
||||
strmapreturn = addSMXToMapReturn(strmapreturn, SMX)
|
||||
}
|
||||
//System.out.println("scoreRelationNewMsgToRecentMsg: " + scoreRelationNewMsgToRecentMsg + "\niterator: " + iterator);
|
||||
iterator++
|
||||
}
|
||||
}
|
||||
return strmapreturn
|
||||
}
|
||||
|
||||
private fun addSMXToMapReturn(strmapreturn: ConcurrentMap<Int?, String?>, SMX: SimilarityMatrix?): ConcurrentMap<Int?, String?> {
|
||||
if (!strmapreturn.containsValue(SMX!!.primaryString)) {
|
||||
strmapreturn[strmapreturn.size] = SMX.primaryString
|
||||
val transmittedStr = SMX.secondaryString
|
||||
val cacheValue1 = SMX.cacheValue1
|
||||
val cacheValue2 = SMX.cacheValue2
|
||||
if (cacheValue1 != null && !sentimentCachingMap.keys.contains(SMX.primaryString)) {
|
||||
sentimentCachingMap[SMX.secondaryString] = SMX.cacheValue1
|
||||
}
|
||||
if (cacheValue2 != null && !sentimentCachingMap.keys.contains(transmittedStr)) {
|
||||
sentimentCachingMap[transmittedStr] = SMX.cacheValue2
|
||||
}
|
||||
}
|
||||
return strmapreturn
|
||||
}
|
||||
|
||||
|
||||
private fun StrComparringNoSentenceRelationMap(strCacheLocal: ConcurrentMap<Int, String?>, strCollection: Collection<String?>, localJMWEMap: ConcurrentMap<String, Annotation>,
|
||||
localPipelineAnnotation: ConcurrentMap<String?, Annotation>, localPipelineSentimentAnnotation: ConcurrentMap<String?, Annotation>,
|
||||
localCoreDocumentMap: ConcurrentMap<String, CoreDocument>): List<SimilarityMatrix?> {
|
||||
|
||||
//TODO here
|
||||
val distance_requirement = 15500.0
|
||||
val prefix_size = 150
|
||||
val smxReturnList: ArrayList<SimilarityMatrix> = ArrayList<SimilarityMatrix>()
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (j in strCollection) {
|
||||
for (i in strCollection) {
|
||||
if (j != i) {
|
||||
val SMXInit = SimilarityMatrix(j, i)
|
||||
val sentimentCacheStr1 = sentimentCachingMap.getOrDefault(i, null)
|
||||
val sentimentCacheStr = sentimentCachingMap.getOrDefault(j, null)
|
||||
var sentimentAnalyzerTest: SentimentAnalyzerTest? = null
|
||||
if (stringCache.size < prefix_size) {
|
||||
sentimentAnalyzerTest = SentimentAnalyzerTest(j, i, SMXInit,
|
||||
localJMWEMap[j], localJMWEMap[i], localPipelineAnnotation[j],
|
||||
localPipelineAnnotation[i], localPipelineSentimentAnnotation[j],
|
||||
localPipelineSentimentAnnotation[i], localCoreDocumentMap[j], localCoreDocumentMap[i],
|
||||
sentimentCacheStr, sentimentCacheStr1)
|
||||
} else {
|
||||
sentimentAnalyzerTest = SentimentAnalyzerTest(j, i, SMXInit,
|
||||
localJMWEMap[j], jmweAnnotationCache[i], localPipelineAnnotation[j],
|
||||
pipelineAnnotationCache[i], localPipelineSentimentAnnotation[j],
|
||||
pipelineSentimentAnnotationCache[i], localCoreDocumentMap[j],
|
||||
coreDocumentAnnotationCache[i], sentimentCacheStr, sentimentCacheStr1)
|
||||
}
|
||||
val call = sentimentAnalyzerTest.call();
|
||||
if (call != null && call.distance > distance_requirement) {
|
||||
smxReturnList.add(call)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
|
||||
return smxReturnList
|
||||
}
|
||||
|
||||
private fun stringIteratorComparator(strmap: ConcurrentMap<Int?, String?>,
|
||||
strCacheLocal: ConcurrentMap<Int, String?>, localJMWEMap: ConcurrentMap<String, Annotation>,
|
||||
localPipelineAnnotation: ConcurrentMap<String?, Annotation>, localPipelineSentimentAnnotation: ConcurrentMap<String?, Annotation>,
|
||||
localCoreDocumentMap: ConcurrentMap<String, CoreDocument>): ConcurrentMap<Int?, String?> {
|
||||
//System.out.println("strmap siuze: " + strmap.size());
|
||||
val ComparringNoSentenceRelationMap: List<SimilarityMatrix> = StrComparringNoSentenceRelationMap(strCacheLocal, strmap.values,
|
||||
localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap) as List<SimilarityMatrix>
|
||||
Collections.sort(ComparringNoSentenceRelationMap, Comparator<SimilarityMatrix> { e1: SimilarityMatrix, e2: SimilarityMatrix -> e1.primaryString.compareTo(e2.primaryString) })
|
||||
//System.out.println("strmapreturn size: " + strmapreturn.size());
|
||||
return futuresReturnOverallEvaluation(ComparringNoSentenceRelationMap)
|
||||
}
|
||||
|
||||
private fun removeNonSensicalStrings(strmap: ConcurrentMap<Int?, String?>): ConcurrentMap<Int?, String?> {
|
||||
val strCacheLocal = stringCache
|
||||
val localJMWEMap = getMultipleJMWEAnnotation(strmap.values)
|
||||
val localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values)
|
||||
val localPipelineSentimentAnnotation = getMultiplePipelineSentimentAnnotation(strmap.values)
|
||||
val localCoreDocumentMap = getMultipleCoreDocumentsWaySuggestion(strmap.values, pipeline)
|
||||
return stringIteratorComparator(strmap, strCacheLocal, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap)
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
@Throws(CustomError::class)
|
||||
fun checkIfUpdateStrings() {
|
||||
if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning) {
|
||||
var str = MessageResponseHandler.getStr()
|
||||
println("str size: " + str.size)
|
||||
str = filterContent(str)
|
||||
str = removeNonSensicalStrings(str)
|
||||
//System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n");
|
||||
str = annotationCacheUpdate(str)
|
||||
println("""
|
||||
annotationCacheUpdate str size POST: ${str.size}
|
||||
|
||||
""".trimIndent())
|
||||
val strf = str
|
||||
if (!stringCache.isEmpty()) {
|
||||
Thread(Runnable {
|
||||
try {
|
||||
DataMapper.InsertMYSQLStrings(strf)
|
||||
} catch (ex: CustomError) {
|
||||
Logger.getLogger(Datahandler::class.java
|
||||
.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
MessageResponseHandler.setStr(MapMaker().concurrencyLevel(6).makeMap())
|
||||
}).start()
|
||||
} else {
|
||||
try {
|
||||
DataMapper.InsertMYSQLStrings(strf)
|
||||
} catch (ex: CustomError) {
|
||||
Logger.getLogger(Datahandler::class.java
|
||||
.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
MessageResponseHandler.setStr(MapMaker().concurrencyLevel(6).makeMap())
|
||||
}
|
||||
if (!stopwatch.isRunning) {
|
||||
stopwatch.start()
|
||||
} else {
|
||||
stopwatch.reset()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun trimString(str: String): String {
|
||||
var str = str
|
||||
str = str.trim { it <= ' ' }
|
||||
if (str.startsWith("<@")) {
|
||||
str = str.substring(str.indexOf("> ") + 2)
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
private fun getResponseFutures(strF: String): String {
|
||||
val values_copy: List<String?> = ArrayList(stringCache.values)
|
||||
Collections.shuffle(values_copy)
|
||||
var preRelationUserCounters = -155000.0
|
||||
val concurrentRelations: MutableList<String?> = arrayListOf()
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (str1 in values_copy) {
|
||||
if (strF != str1) {
|
||||
val sentimentCacheStr1 = sentimentCachingMap.getOrDefault(str1, null)
|
||||
val worker: Callable<SimilarityMatrix> = SentimentAnalyzerTest(strF, str1, SimilarityMatrix(strF, str1),
|
||||
strAnnoJMWE, jmweAnnotationCache[str1], strAnno,
|
||||
pipelineAnnotationCache[str1], strAnnoSentiment,
|
||||
pipelineSentimentAnnotationCache[str1], coreDoc, coreDocumentAnnotationCache[str1],
|
||||
null, sentimentCacheStr1)
|
||||
try {
|
||||
val getSMX = worker.call()
|
||||
if (getSMX != null) {
|
||||
val scoreRelationLastUserMsg = getSMX.distance
|
||||
if (scoreRelationLastUserMsg > preRelationUserCounters) {
|
||||
preRelationUserCounters = scoreRelationLastUserMsg
|
||||
concurrentRelations.add(getSMX.secondaryString)
|
||||
}
|
||||
}
|
||||
} catch (ex: Exception) {
|
||||
Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
}
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
|
||||
val SB = StringBuilder()
|
||||
val randomLenghtPermit = strF.length * (Math.random() * Math.random() * Math.random() * 5)
|
||||
Collections.reverse(concurrentRelations)
|
||||
if (concurrentRelations.isEmpty()) {
|
||||
return "failure, preventing stuckness"
|
||||
}
|
||||
|
||||
val firstRelation = concurrentRelations[0]
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (secondaryRelation in concurrentRelations) {
|
||||
if (SB.toString().length > randomLenghtPermit && !SB.toString().isEmpty()) {
|
||||
break
|
||||
}
|
||||
val append = appendToString(firstRelation, secondaryRelation)
|
||||
if (append) {
|
||||
SB.append(secondaryRelation).append(" ")
|
||||
}
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
return SB.toString()
|
||||
}
|
||||
|
||||
private fun appendToString(firstRelation: String?, secondaryRelation: String?): Boolean {
|
||||
if (firstRelation == secondaryRelation) {
|
||||
return true
|
||||
}
|
||||
val scoreRelationStrF = getScoreRelationStrF(firstRelation, secondaryRelation)
|
||||
return if (scoreRelationStrF > 1900) {
|
||||
true
|
||||
} else false
|
||||
}
|
||||
|
||||
@Throws(CustomError::class)
|
||||
fun getResponseMsg(str: String): String {
|
||||
val strF = trimString(str)
|
||||
getSingularAnnotation(strF)
|
||||
return getResponseFutures(strF)
|
||||
}
|
||||
|
||||
fun getSingularAnnotation(str: String?) {
|
||||
strAnno = Annotation(str)
|
||||
strAnno!!.compact()
|
||||
pipeline.annotate(strAnno)
|
||||
strAnnoSentiment = Annotation(str)
|
||||
strAnnoSentiment!!.compact()
|
||||
pipelineSentiment!!.annotate(strAnnoSentiment)
|
||||
val notactualList: MutableList<String?> = arrayListOf()
|
||||
notactualList.add(str)
|
||||
val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(notactualList)
|
||||
strAnnoJMWE = jmweAnnotation.values.iterator().next()
|
||||
strAnnoJMWE.compact()
|
||||
val coreDocument = CoreDocument(str)
|
||||
pipeline.annotate(coreDocument)
|
||||
coreDoc = coreDocument
|
||||
}
|
||||
|
||||
private fun getScoreRelationStrF(str: String?, mostRecentMsg: String?): Double {
|
||||
val SMX = SimilarityMatrix(str, mostRecentMsg)
|
||||
val cacheSentiment1 = sentimentCachingMap.getOrDefault(str, null)
|
||||
val cacheSentiment2 = sentimentCachingMap.getOrDefault(mostRecentMsg, null)
|
||||
val worker: Callable<SimilarityMatrix> = SentimentAnalyzerTest(str, mostRecentMsg, SMX,
|
||||
strAnnoJMWE, jmweAnnotationCache[mostRecentMsg], strAnno,
|
||||
pipelineAnnotationCache[mostRecentMsg], strAnnoSentiment,
|
||||
pipelineSentimentAnnotationCache[mostRecentMsg], coreDoc, coreDocumentAnnotationCache[mostRecentMsg], cacheSentiment1, cacheSentiment2)
|
||||
var callSMX: SimilarityMatrix? = null
|
||||
try {
|
||||
callSMX = worker.call()
|
||||
} catch (ex: Exception) {
|
||||
Logger.getLogger(Datahandler::class.java
|
||||
.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
return callSMX?.distance ?: 0.0
|
||||
}
|
||||
|
||||
private fun annotationCacheUpdate(strmap: ConcurrentMap<Int?, String?>): ConcurrentMap<Int?, String?> {
|
||||
val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strmap.values)
|
||||
for ((key, value) in jmweAnnotation) {
|
||||
jmweAnnotationCache[key] = value
|
||||
}
|
||||
val Annotationspipeline = MapMaker().concurrencyLevel(6).makeMap<String?, Annotation>()
|
||||
val AnnotationspipelineSentiment = MapMaker().concurrencyLevel(6).makeMap<String?, Annotation>()
|
||||
val coreDocumentpipelineMap = getMultipleCoreDocumentsWaySuggestion(strmap.values, pipeline)
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (str in strmap.values) {
|
||||
val strAnno1 = Annotation(str)
|
||||
Annotationspipeline[str] = strAnno1
|
||||
val strAnno2 = Annotation(str)
|
||||
AnnotationspipelineSentiment[str] = strAnno2
|
||||
stringCache[stringCache.size + 1] = str
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
pipeline.annotate(Annotationspipeline.values)
|
||||
pipelineSentiment!!.annotate(AnnotationspipelineSentiment.values)
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (pipelineEntry in Annotationspipeline.entries) {
|
||||
if (pipelineEntry != null) {
|
||||
pipelineAnnotationCache[pipelineEntry.key] = pipelineEntry.value
|
||||
}
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (pipelineEntry in AnnotationspipelineSentiment.entries) {
|
||||
if (pipelineEntry != null) {
|
||||
pipelineSentimentAnnotationCache[pipelineEntry.key] = pipelineEntry.value
|
||||
}
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (coreDocumentEntry in coreDocumentpipelineMap.entries) {
|
||||
coreDocumentAnnotationCache[coreDocumentEntry.key] = coreDocumentEntry.value
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
return strmap
|
||||
}
|
||||
|
||||
val messageOverHead: Int
|
||||
get() = stringCache.values.size - stringCache.values.size / 10
|
||||
|
||||
fun update_autismo_socket_msg() {
|
||||
try {
|
||||
try {
|
||||
DatagramSocket(48480).use { serverSocket ->
|
||||
DatagramSocket(48471).use { serverSocket1 ->
|
||||
val receiveData = ByteArray(4096)
|
||||
val IPAddress = InetAddress.getByName("135.125.188.157") //later moving autism bot 1 and 3 to OVH from SYS
|
||||
var receivePacket = DatagramPacket(receiveData, receiveData.size)
|
||||
while (true) {
|
||||
serverSocket.receive(receivePacket)
|
||||
var sentence = String(receivePacket.data, 0, receivePacket.length)
|
||||
sentence = sentence.replace("clientmessage:", "")
|
||||
var getResponseMsg = getResponseMsg(sentence)
|
||||
var sendData = getResponseMsg.toByteArray(charset("UTF-8"))
|
||||
var sendPacket = DatagramPacket(sendData, sendData.size, IPAddress, 48479)
|
||||
serverSocket.send(sendPacket)
|
||||
receivePacket = DatagramPacket(receiveData, receiveData.size)
|
||||
serverSocket1.receive(receivePacket)
|
||||
sentence = String(receivePacket.data, 0, receivePacket.length)
|
||||
sentence = sentence.replace("clientmessage:", "")
|
||||
getResponseMsg = getResponseMsg(sentence)
|
||||
sendData = getResponseMsg.toByteArray(charset("UTF-8"))
|
||||
sendPacket = DatagramPacket(sendData, sendData.size, IPAddress, 48476)
|
||||
serverSocket1.send(sendPacket)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (ex: CustomError) {
|
||||
Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
} catch (ex: SocketException) {
|
||||
Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex)
|
||||
} catch (ex: UnsupportedEncodingException) {
|
||||
Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex)
|
||||
} catch (ex: IOException) {
|
||||
Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
}
|
||||
|
||||
private class AnnotationCollector<T> : Consumer<T> {
|
||||
val annotationsT: MutableList<T?> = arrayListOf()
|
||||
override fun accept(ann: T) {
|
||||
//System.out.println("adding ann: " + ann.toString());
|
||||
annotationsT.add(ann)
|
||||
}
|
||||
|
||||
companion object {
|
||||
var i = 0
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
val EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(10, TimeUnit.MINUTES)
|
||||
val EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS)
|
||||
|
||||
@JvmField
|
||||
var instance = Datahandler()
|
||||
private var strAnno: Annotation? = null
|
||||
private var strAnnoSentiment: Annotation? = null
|
||||
private lateinit var strAnnoJMWE: Annotation
|
||||
private var coreDoc: CoreDocument? = null
|
||||
private val stringCache = MapMaker().concurrencyLevel(6).makeMap<Int, String?>()
|
||||
private lateinit var pipelineAnnotationCache: ConcurrentMap<String?, Annotation>
|
||||
private lateinit var pipelineSentimentAnnotationCache: ConcurrentMap<String?, Annotation>
|
||||
private lateinit var jmweAnnotationCache: ConcurrentMap<String, Annotation>
|
||||
private lateinit var coreDocumentAnnotationCache: ConcurrentMap<String, CoreDocument>
|
||||
private val sentimentCachingMap = MapMaker().concurrencyLevel(6).makeMap<String, SentimentValueCache>()
|
||||
private const val similar = ""
|
||||
private const val shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz"
|
||||
private const val sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz"
|
||||
private const val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
|
||||
private const val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger"
|
||||
private const val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz"
|
||||
private const val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz"
|
||||
private const val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz"
|
||||
private const val customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"
|
||||
|
||||
@JvmStatic
|
||||
var tagger: MaxentTagger? = null
|
||||
private set
|
||||
private val options = arrayOf("-maxLength", "100")
|
||||
private val props = Properties()
|
||||
private val propsSentiment = Properties()
|
||||
|
||||
@JvmStatic
|
||||
var gsf: GrammaticalStructureFactory? = null
|
||||
private set
|
||||
private lateinit var lp: LexicalizedParser
|
||||
private lateinit var tlp: TreebankLanguagePack
|
||||
private lateinit var classifier: AbstractSequenceClassifier<CoreLabel>
|
||||
|
||||
public fun getPipeLine(): StanfordCoreNLP {
|
||||
return pipeline
|
||||
}
|
||||
|
||||
// set up Stanford CoreNLP pipeline
|
||||
@JvmStatic
|
||||
val pipeline = pipeLineSetUp
|
||||
private var pipelineSentiment: StanfordCoreNLP? = null
|
||||
|
||||
private val pipeLineSetUp: StanfordCoreNLP
|
||||
private get() {
|
||||
props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse")
|
||||
props.setProperty("parse.model", shiftReduceParserPath)
|
||||
props.setProperty("parse.maxlen", "90")
|
||||
props.setProperty("parse.binaryTrees", "true")
|
||||
props.setProperty("threads", "5")
|
||||
props.setProperty("pos.maxlen", "90")
|
||||
props.setProperty("tokenize.maxlen", "90")
|
||||
props.setProperty("ssplit.maxlen", "90")
|
||||
props.setProperty("lemma.maxlen", "90")
|
||||
props.setProperty("ner.model", "$nerModel,$nerModel2,$nerModel3")
|
||||
props.setProperty("ner.combinationMode", "HIGH_RECALL")
|
||||
props.setProperty("regexner.ignorecase", "true")
|
||||
props.setProperty("ner.fine.regexner.ignorecase", "true")
|
||||
props.setProperty("tokenize.options", "untokenizable=firstDelete")
|
||||
return StanfordCoreNLP(props)
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun getClassifier(): AbstractSequenceClassifier<CoreLabel>? {
|
||||
return classifier
|
||||
}
|
||||
|
||||
fun setClassifier(classifier: AbstractSequenceClassifier<CoreLabel>?) {
|
||||
if (classifier != null) {
|
||||
Companion.classifier = classifier
|
||||
}
|
||||
}
|
||||
|
||||
private fun getMultipleJMWEAnnotation(str: Collection<String?>): ConcurrentMap<String, Annotation> {
|
||||
return PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str)
|
||||
}
|
||||
|
||||
private fun getMultiplePipelineAnnotation(str: Collection<String?>): ConcurrentMap<String?, Annotation> {
|
||||
val pipelineAnnotationMap = MapMaker().concurrencyLevel(2).makeMap<String?, Annotation>()
|
||||
for (str1 in str) {
|
||||
val strAnno1 = Annotation(str1)
|
||||
pipelineAnnotationMap[str1] = strAnno1
|
||||
}
|
||||
pipeline.annotate(pipelineAnnotationMap.values)
|
||||
return pipelineAnnotationMap
|
||||
}
|
||||
|
||||
private fun getMultiplePipelineSentimentAnnotation(str: Collection<String?>): ConcurrentMap<String?, Annotation> {
|
||||
val pipelineAnnotationMap = MapMaker().concurrencyLevel(2).makeMap<String?, Annotation>()
|
||||
for (str1 in str) {
|
||||
val strAnno1 = Annotation(str1)
|
||||
pipelineAnnotationMap[str1] = strAnno1
|
||||
}
|
||||
pipelineSentiment!!.annotate(pipelineAnnotationMap.values)
|
||||
return pipelineAnnotationMap
|
||||
}
|
||||
|
||||
fun filterContent(str: ConcurrentMap<Int?, String?>): ConcurrentMap<Int?, String?> {
|
||||
val strlistreturn = MapMaker().concurrencyLevel(2).makeMap<Int?, String?>()
|
||||
str.values.forEach(Consumer {
|
||||
var str1: String = ""
|
||||
if (!str1.isEmpty() && str1.length > 3) {
|
||||
str1 = str1.trim { it <= ' ' }
|
||||
if (str1.contains("PM*")) {
|
||||
str1 = str1.substring(str1.indexOf("PM*") + 3)
|
||||
}
|
||||
if (str1.contains("AM*")) {
|
||||
str1 = str1.substring(str1.indexOf("AM*") + 3)
|
||||
}
|
||||
/*
|
||||
if (str1.contains("?") || str1.contains("°"))
|
||||
{
|
||||
if (!str1.contains("http"))
|
||||
{
|
||||
str1 = str1.replace("?", " <:wlenny:514861023002624001> ");
|
||||
str1 = str1.replace("°", " <:wlenny:514861023002624001> ");
|
||||
}
|
||||
}
|
||||
*/if (str1.contains("(Counter-Terrorist)")) {
|
||||
str1 = str1.replace("(Counter-Terrorist)", " ")
|
||||
}
|
||||
if (str1.contains("(Terrorist)")) {
|
||||
str1 = str1.replace("(Terrorist)", " ")
|
||||
}
|
||||
if (str1.contains("(Spectator)")) {
|
||||
str1 = str1.replace("(Spectator)", " ")
|
||||
}
|
||||
if (str1.contains("*DEAD*")) {
|
||||
str1 = str1.replace("*DEAD*", " ")
|
||||
}
|
||||
if (str1.contains("{red}")) {
|
||||
str1 = str1.replace("{red}", " ")
|
||||
}
|
||||
if (str1.contains("{orange}")) {
|
||||
str1 = str1.replace("{orange}", " ")
|
||||
}
|
||||
if (str1.contains("{yellow}")) {
|
||||
str1 = str1.replace("{yellow}", " ")
|
||||
}
|
||||
if (str1.contains("{green}")) {
|
||||
str1 = str1.replace("{green}", " ")
|
||||
}
|
||||
if (str1.contains("{lightblue}")) {
|
||||
str1 = str1.replace("{lightblue}", " ")
|
||||
}
|
||||
if (str1.contains("{blue}")) {
|
||||
str1 = str1.replace("{blue}", " ")
|
||||
}
|
||||
if (str1.contains("{purple}")) {
|
||||
str1 = str1.replace("{purple}", " ")
|
||||
}
|
||||
if (str1.contains("{white}")) {
|
||||
str1 = str1.replace("{white}", " ")
|
||||
}
|
||||
if (str1.contains("{fullblue}")) {
|
||||
str1 = str1.replace("{fullblue}", " ")
|
||||
}
|
||||
if (str1.contains("{cyan}")) {
|
||||
str1 = str1.replace("{cyan}", " ")
|
||||
}
|
||||
if (str1.contains("{lime}")) {
|
||||
str1 = str1.replace("{lime}", " ")
|
||||
}
|
||||
if (str1.contains("{deeppink}")) {
|
||||
str1 = str1.replace("{deeppink}", " ")
|
||||
}
|
||||
if (str1.contains("{slategray}")) {
|
||||
str1 = str1.replace("{slategray}", " ")
|
||||
}
|
||||
if (str1.contains("{dodgerblue}")) {
|
||||
str1 = str1.replace("{dodgerblue}", " ")
|
||||
}
|
||||
if (str1.contains("{black}")) {
|
||||
str1 = str1.replace("{black}", " ")
|
||||
}
|
||||
if (str1.contains("{orangered}")) {
|
||||
str1 = str1.replace("{orangered}", " ")
|
||||
}
|
||||
if (str1.contains("{darkorchid}")) {
|
||||
str1 = str1.replace("{darkorchid}", " ")
|
||||
}
|
||||
if (str1.contains("{pink}")) {
|
||||
str1 = str1.replace("{pink}", " ")
|
||||
}
|
||||
if (str1.contains("{lightyellow}")) {
|
||||
str1 = str1.replace("{lightyellow}", " ")
|
||||
}
|
||||
if (str1.contains("{chocolate}")) {
|
||||
str1 = str1.replace("{chocolate}", " ")
|
||||
}
|
||||
if (str1.contains("{beige}")) {
|
||||
str1 = str1.replace("{beige}", " ")
|
||||
}
|
||||
if (str1.contains("{azure}")) {
|
||||
str1 = str1.replace("{azure}", " ")
|
||||
}
|
||||
if (str1.contains("{yellowgreen}")) {
|
||||
str1 = str1.replace("{yellowgreen}", " ")
|
||||
}
|
||||
str1 = str1.trim { it <= ' ' }
|
||||
if (str1.length > 2 && !str1.startsWith("!")) {
|
||||
strlistreturn[strlistreturn.size] = str1
|
||||
}
|
||||
}
|
||||
})
|
||||
return strlistreturn
|
||||
}
|
||||
|
||||
fun getMultipleCoreDocumentsWaySuggestion(str: Collection<String?>, localNLP: StanfordCoreNLP): ConcurrentMap<String, CoreDocument> {
|
||||
val annCollector: AnnotationCollector<Annotation?> = AnnotationCollector<Annotation?>()
|
||||
val annotationreturnMap = MapMaker().concurrencyLevel(6).makeMap<String, CoreDocument>()
|
||||
runBlocking {
|
||||
val job = launch(Dispatchers.Default) {
|
||||
for (exampleString in str) {
|
||||
localNLP.annotate(Annotation(exampleString), annCollector)
|
||||
AnnotationCollector.i++
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job.join()
|
||||
}
|
||||
try {
|
||||
Thread.sleep(1500)
|
||||
} catch (ex: InterruptedException) {
|
||||
Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
runBlocking {
|
||||
val job1 = launch(Dispatchers.Default) {
|
||||
for (ann in annCollector.annotationsT) {
|
||||
if (ann != null) {
|
||||
ann.compact()
|
||||
val CD = CoreDocument(ann)
|
||||
annotationreturnMap[CD.text()] = CD
|
||||
}
|
||||
}
|
||||
yield()
|
||||
}
|
||||
job1.join()
|
||||
}
|
||||
try {
|
||||
Thread.sleep(1500)
|
||||
} catch (ex: InterruptedException) {
|
||||
Logger.getLogger(Datahandler::class.java.name).log(Level.SEVERE, null, ex)
|
||||
}
|
||||
return annotationreturnMap
|
||||
}
|
||||
}
|
||||
|
||||
init {
|
||||
stopwatch = Stopwatch.createUnstarted()
|
||||
jmweAnnotationCache = MapMaker().concurrencyLevel(4).makeMap<String, Annotation>()
|
||||
pipelineAnnotationCache = MapMaker().concurrencyLevel(4).makeMap<String, Annotation>()
|
||||
pipelineSentimentAnnotationCache = MapMaker().concurrencyLevel(4).makeMap<String, Annotation>()
|
||||
coreDocumentAnnotationCache = MapMaker().concurrencyLevel(5).makeMap<String, CoreDocument>()
|
||||
}
|
||||
}
|
||||
@@ -44,16 +44,8 @@ public class MessageResponseHandler {
|
||||
}
|
||||
|
||||
public static String selectReponseMessage(String toString, String personName) throws CustomError {
|
||||
ConcurrentMap<Integer, String> str1 = new MapMaker().concurrencyLevel(6).makeMap();
|
||||
str1.put(str1.size() + 1, toString);
|
||||
String strreturn = "";
|
||||
for (String str : str1.values()) {
|
||||
if (!str.isEmpty()) {
|
||||
strreturn = str;
|
||||
}
|
||||
}
|
||||
String getResponseMsg = Datahandler.instance.getResponseMsg(strreturn);
|
||||
getResponseMsg = checkPersonPresentInSentence(personName, getResponseMsg, strreturn);
|
||||
String getResponseMsg = Datahandler.instance.getResponseMsg(toString);
|
||||
getResponseMsg = checkPersonPresentInSentence(personName, getResponseMsg, toString);
|
||||
return getResponseMsg;
|
||||
}
|
||||
|
||||
@@ -66,6 +58,8 @@ public class MessageResponseHandler {
|
||||
CoreDocument pipelineCoreDcoumentLastMsg = new CoreDocument(userLastMessage);
|
||||
Datahandler.getPipeline().annotate(pipelineCoreDcoument);
|
||||
Datahandler.getPipeline().annotate(pipelineCoreDcoumentLastMsg);
|
||||
//Datahandler.pipeline.annotate(pipelineCoreDcoument);
|
||||
//Datahandler.pipeline.annotate(pipelineCoreDcoumentLastMsg);
|
||||
String regex = "(.*?\\d){10,}";
|
||||
for (CoreEntityMention em : pipelineCoreDcoument.entityMentions()) {
|
||||
String entityType = em.entityType();
|
||||
|
||||
@@ -54,7 +54,7 @@ public class PipelineJMWESingleton {
|
||||
public final ConcurrentMap<String, Annotation> getJMWEAnnotation(Collection<String> strvalues) {
|
||||
boolean verbose = false;
|
||||
IMWEIndex index;
|
||||
String jmweIndexData = "/home/debian/autism_bot/lib/mweindex_wordnet3.0_semcor1.6.data"; // ./lib/mweindex_wordnet3.0_semcor1.6.data
|
||||
String jmweIndexData = "/home/gameservers/autism_bot/lib/mweindex_wordnet3.0_semcor1.6.data"; // ./lib/mweindex_wordnet3.0_semcor1.6.data
|
||||
String jmweIndexDataLocalTest = "E:/java8/Projects/mweindex_wordnet3.0_semcor1.6.data";
|
||||
File indexFile = new File((String) jmweIndexData);
|
||||
index = new MWEIndex(indexFile);
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
Manifest-Version: 1.0
|
||||
Main-Class: PresentationLayer.DiscordHandler
|
||||
|
||||
@@ -37,7 +37,6 @@ import java.math.BigInteger;
|
||||
*/
|
||||
public class DiscordHandler {
|
||||
public static void main(String[] args) {
|
||||
System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", "15");
|
||||
try {
|
||||
Datahandler.instance.initiateMYSQL();
|
||||
//nohup screen -d -m -S nonroot java -Xmx6900M -jar /home/javatests/ArtificialAutism-1.0.jar
|
||||
@@ -51,7 +50,6 @@ public class DiscordHandler {
|
||||
Datahandler.instance.shiftReduceParserInitiate();
|
||||
Datahandler.instance.instantiateAnnotationMap();
|
||||
System.out.println("FINISHED ALL ANNOTATIONS");
|
||||
Datahandler.instance.addHLstatsMessages();
|
||||
Datahandler.instance.updateStringCache();
|
||||
//String token = "NTI5NzAxNTk5NjAyMjc4NDAx.Dw0vDg.7-aMjVWdQMYPl8qVNyvTCPS5F_A";
|
||||
String token = new settings().getDiscordToken();
|
||||
|
||||
Reference in New Issue
Block a user