redid kotlin to java and removed coroutines again
This commit is contained in:
christian
parent
8eacf7da71
commit
d7ed45922e
@ -20,7 +20,7 @@ import java.util.logging.Logger;
|
|||||||
*/
|
*/
|
||||||
public class DataMapper {
|
public class DataMapper {
|
||||||
|
|
||||||
public static ArrayList<String> getAllStrings() throws SQLException {
|
public static ArrayList<String> getAllStrings() {
|
||||||
Connection l_cCon = null;
|
Connection l_cCon = null;
|
||||||
PreparedStatement l_pStatement = null;
|
PreparedStatement l_pStatement = null;
|
||||||
ResultSet l_rsSearch = null;
|
ResultSet l_rsSearch = null;
|
||||||
@ -33,13 +33,15 @@ public class DataMapper {
|
|||||||
while (l_rsSearch.next()) {
|
while (l_rsSearch.next()) {
|
||||||
arrayListStr.add(l_rsSearch.getString(1));
|
arrayListStr.add(l_rsSearch.getString(1));
|
||||||
}
|
}
|
||||||
|
} catch (SQLException throwables) {
|
||||||
|
throwables.printStackTrace();
|
||||||
} finally {
|
} finally {
|
||||||
CloseConnections(l_pStatement, l_rsSearch, l_cCon);
|
CloseConnections(l_pStatement, l_rsSearch, l_cCon);
|
||||||
}
|
}
|
||||||
return arrayListStr;
|
return arrayListStr;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void InsertMYSQLStrings(String str) throws SQLException {
|
public static void InsertMYSQLStrings(String str) {
|
||||||
Connection l_cCon = null;
|
Connection l_cCon = null;
|
||||||
PreparedStatement l_pStatement = null;
|
PreparedStatement l_pStatement = null;
|
||||||
ResultSet l_rsSearch = null;
|
ResultSet l_rsSearch = null;
|
||||||
@ -49,6 +51,8 @@ public class DataMapper {
|
|||||||
l_pStatement = l_cCon.prepareStatement(l_sSQL);
|
l_pStatement = l_cCon.prepareStatement(l_sSQL);
|
||||||
l_pStatement.setString(1, str);
|
l_pStatement.setString(1, str);
|
||||||
l_pStatement.execute();
|
l_pStatement.execute();
|
||||||
|
} catch (SQLException throwables) {
|
||||||
|
throwables.printStackTrace();
|
||||||
} finally {
|
} finally {
|
||||||
CloseConnections(l_pStatement, l_rsSearch, l_cCon);
|
CloseConnections(l_pStatement, l_rsSearch, l_cCon);
|
||||||
}
|
}
|
||||||
|
|||||||
677
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
Normal file
677
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
Normal file
@ -0,0 +1,677 @@
|
|||||||
|
package FunctionLayer;
|
||||||
|
|
||||||
|
import DataLayer.DataMapper;
|
||||||
|
import FunctionLayer.StanfordParser.SentimentAnalyzerTest;
|
||||||
|
import edu.mit.jmwe.data.IMWE;
|
||||||
|
import edu.mit.jmwe.data.IToken;
|
||||||
|
import edu.stanford.nlp.ie.AbstractSequenceClassifier;
|
||||||
|
import edu.stanford.nlp.ie.crf.CRFClassifier;
|
||||||
|
import edu.stanford.nlp.ie.machinereading.structure.EntityMention;
|
||||||
|
import edu.stanford.nlp.ling.CoreAnnotations;
|
||||||
|
import edu.stanford.nlp.ling.CoreLabel;
|
||||||
|
import edu.stanford.nlp.ling.TaggedWord;
|
||||||
|
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
|
||||||
|
import edu.stanford.nlp.pipeline.Annotation;
|
||||||
|
import edu.stanford.nlp.pipeline.CoreDocument;
|
||||||
|
import edu.stanford.nlp.pipeline.CoreEntityMention;
|
||||||
|
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
||||||
|
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
|
||||||
|
import edu.stanford.nlp.trees.*;
|
||||||
|
import edu.stanford.nlp.util.CoreMap;
|
||||||
|
import kotlinx.coroutines.*;
|
||||||
|
import org.ejml.simple.SimpleMatrix;
|
||||||
|
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
|
||||||
|
public class Datahandler {
|
||||||
|
private HashMap<String, Annotation> pipelineAnnotationCache;
|
||||||
|
private HashMap<String, Annotation> pipelineSentimentAnnotationCache;
|
||||||
|
private HashMap<String, CoreDocument> coreDocumentAnnotationCache;
|
||||||
|
private HashMap<String, Annotation> jmweAnnotationCache;
|
||||||
|
|
||||||
|
private MaxentTagger tagger = new MaxentTagger();
|
||||||
|
|
||||||
|
private GrammaticalStructureFactory gsf;
|
||||||
|
private AbstractSequenceClassifier<CoreLabel> classifier;
|
||||||
|
|
||||||
|
//SentimentAnalyzer Hashmaps
|
||||||
|
private HashMap<String, Integer> tokenizeCountingHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, List<List<TaggedWord>>> taggedWordListHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> retrieveTGWListHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, List<CoreMap>> sentences1HashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, List<CoreMap>> sentencesSentimentHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<Tree>> trees1HashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<GrammaticalStructure>> grammaticalStructureHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<TypedDependency>> typedDependenciesHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<Integer>> rnnCoreAnnotationsPredictedHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<SimpleMatrix>> simpleMatricesHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<SimpleMatrix>> simpleMatricesNodevectorsHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, List> listHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, Integer> longestHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, Integer> sentimentHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, List<IMWE<IToken>>> imwesHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, Integer> InflectedCounterNegativeHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, Integer> InflectedCounterPositiveHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, ArrayList<String>> tokenEntryHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, Integer> MarkedContinuousCounterHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, Integer> UnmarkedPatternCounterHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> strTokensIpartFormHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> tokenFormsHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> strTokenEntryGetPOSHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<Integer>> intTokenEntyCountsHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> ITokenTagsHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> strTokenStemsHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, Integer> AnotatorcounterHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, Integer> TokensCounterHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> entityTokenTagsHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> nerEntitiesHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> nerEntitiesTypeHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> stopWordTokenHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, java.util.ArrayList<String>> stopWordLemmaHashMap = new HashMap();
|
||||||
|
|
||||||
|
private HashMap<String, Integer> PairCounterHashMap = new HashMap();
|
||||||
|
|
||||||
|
public Datahandler() {
|
||||||
|
jmweAnnotationCache = new HashMap<String, Annotation>();
|
||||||
|
pipelineAnnotationCache = new HashMap<String, Annotation>();
|
||||||
|
pipelineSentimentAnnotationCache = new HashMap<String, Annotation>();
|
||||||
|
coreDocumentAnnotationCache = new HashMap<String, CoreDocument>();
|
||||||
|
gsf = initiateGrammaticalStructureFactory();
|
||||||
|
String nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz";
|
||||||
|
classifier = CRFClassifier.getClassifierNoExceptions(nerModel);
|
||||||
|
}
|
||||||
|
|
||||||
|
private GrammaticalStructureFactory initiateGrammaticalStructureFactory() {
|
||||||
|
// lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
|
||||||
|
String lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";
|
||||||
|
LexicalizedParser lp = LexicalizedParser.loadModel(lexParserEnglishPCFG, "-maxLength", "100");
|
||||||
|
TreebankLanguagePack tlp = lp.getOp().langpack();
|
||||||
|
return tlp.grammaticalStructureFactory();
|
||||||
|
}
|
||||||
|
|
||||||
|
public StanfordCoreNLP pipeLineSetUp() {
|
||||||
|
Properties props = new Properties();
|
||||||
|
String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz";
|
||||||
|
// nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz"
|
||||||
|
//String nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz";
|
||||||
|
// nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz"
|
||||||
|
//String nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.distsim.crf.ser.gz";
|
||||||
|
props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse");
|
||||||
|
props.setProperty("parse.model", shiftReduceParserPath);
|
||||||
|
props.setProperty("parse.maxlen", "90");
|
||||||
|
props.setProperty("parse.binaryTrees", "true");
|
||||||
|
props.setProperty("threads", "5");
|
||||||
|
props.setProperty("pos.maxlen", "90");
|
||||||
|
props.setProperty("tokenize.maxlen", "90");
|
||||||
|
props.setProperty("ssplit.maxlen", "90");
|
||||||
|
props.setProperty("lemma.maxlen", "90");
|
||||||
|
props.setProperty("ner.model", "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz" +
|
||||||
|
",edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz" +
|
||||||
|
",edu/stanford/nlp/models/ner/english.muc.7class.distsim.crf.ser.gz");
|
||||||
|
props.setProperty("ner.combinationMode", "HIGH_RECALL");
|
||||||
|
props.setProperty("regexner.ignorecase", "true");
|
||||||
|
props.setProperty("ner.fine.regexner.ignorecase", "true");
|
||||||
|
props.setProperty("tokenize.options", "untokenizable=firstKeep");
|
||||||
|
return new StanfordCoreNLP(props);
|
||||||
|
}
|
||||||
|
|
||||||
|
public StanfordCoreNLP shiftReduceParserInitiate() {
|
||||||
|
Properties propsSentiment = new Properties();
|
||||||
|
// lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
|
||||||
|
String lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";
|
||||||
|
String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz";
|
||||||
|
// taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger"
|
||||||
|
String taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words-distsim.tagger";
|
||||||
|
String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of," +
|
||||||
|
"on,or,such,that,the,their,then,there,these,they,this,to,was,will,with";
|
||||||
|
propsSentiment.setProperty("parse.model", lexParserEnglishPCFG);
|
||||||
|
propsSentiment.setProperty("sentiment.model", sentimentModel);
|
||||||
|
propsSentiment.setProperty("parse.maxlen", "90");
|
||||||
|
propsSentiment.setProperty("threads", "5");
|
||||||
|
propsSentiment.setProperty("pos.maxlen", "90");
|
||||||
|
propsSentiment.setProperty("tokenize.maxlen", "90");
|
||||||
|
propsSentiment.setProperty("ssplit.maxlen", "90");
|
||||||
|
propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword"); //coref too expensive memorywise
|
||||||
|
propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator");
|
||||||
|
propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList);
|
||||||
|
propsSentiment.setProperty("tokenize.options", "untokenizable=firstKeep");
|
||||||
|
tagger = new MaxentTagger(taggerPath);
|
||||||
|
return new StanfordCoreNLP(propsSentiment);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String trimString(String str) {
|
||||||
|
String message = str.trim();
|
||||||
|
if (message.startsWith("<@")) {
|
||||||
|
message = message.substring(message.indexOf("> ") + 2);
|
||||||
|
}
|
||||||
|
if (!message.isEmpty()) {
|
||||||
|
message = message.replace("@", "");
|
||||||
|
if (message.contains("<>")) {
|
||||||
|
message = message.substring(message.indexOf(">"));
|
||||||
|
}
|
||||||
|
if (message.startsWith("[ *")) {
|
||||||
|
message = message.substring(message.indexOf("]"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createStrAnnotation(String str, StanfordCoreNLP stanfordCoreNLP, Boolean sentimentBool) {
|
||||||
|
Annotation strAnno2 = new Annotation(str);
|
||||||
|
strAnno2.compact();
|
||||||
|
stanfordCoreNLP.annotate(strAnno2);
|
||||||
|
if (sentimentBool) {
|
||||||
|
pipelineSentimentAnnotationCache.put(str, strAnno2);
|
||||||
|
} else {
|
||||||
|
pipelineAnnotationCache.put(str, strAnno2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getResponseFutures(String strF, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) {
|
||||||
|
|
||||||
|
Annotation strAnno = new Annotation(strF);
|
||||||
|
strAnno.compact();
|
||||||
|
stanfordCoreNLP.annotate(strAnno);
|
||||||
|
|
||||||
|
Annotation strAnnoSentiment = new Annotation(strF);
|
||||||
|
strAnnoSentiment.compact();
|
||||||
|
stanfordCoreNLPSentiment.annotate(strAnnoSentiment);
|
||||||
|
|
||||||
|
Annotation annotation = new Annotation(strF);
|
||||||
|
stanfordCoreNLP.annotate(annotation);
|
||||||
|
CoreDocument coreDocument = new CoreDocument(annotation);
|
||||||
|
|
||||||
|
List<String> ues_copy = new ArrayList(DataMapper.getAllStrings());
|
||||||
|
double preRelationUserCounters = -155000.0;
|
||||||
|
ArrayList<String> concurrentRelations = new ArrayList();
|
||||||
|
StringBuilder SB = new StringBuilder();
|
||||||
|
Annotation jmweAnnotationF = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strF);
|
||||||
|
Integer tokenizeCountingF = null;
|
||||||
|
List<List<TaggedWord>> taggedWordListF = null;
|
||||||
|
java.util.ArrayList<String> retrieveTGWListF = null;
|
||||||
|
List<CoreMap> sentencesF = null;
|
||||||
|
List<CoreMap> sentencesSentimentF = null;
|
||||||
|
|
||||||
|
List<CoreMap> coreMaps1 = jmweAnnotationF.get(CoreAnnotations.SentencesAnnotation.class);
|
||||||
|
|
||||||
|
java.util.ArrayList<Tree> treesF = null;
|
||||||
|
|
||||||
|
ArrayList<GrammaticalStructure> grammaticalStructuresF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<TypedDependency> typedDependenciesF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<Integer> rnnCoreAnnotationsPredictedF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<SimpleMatrix> simpleMatricesF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<SimpleMatrix> simpleMatricesNodevectorsF = null;
|
||||||
|
List<String> listF = null;
|
||||||
|
|
||||||
|
Integer longestF = null;
|
||||||
|
|
||||||
|
Integer sentimentLongestF = null;
|
||||||
|
|
||||||
|
List<IMWE<IToken>> imwesF = null;
|
||||||
|
|
||||||
|
Integer InflectedCounterNegativeF = null;
|
||||||
|
|
||||||
|
Integer InflectedCounterPositiveF = null;
|
||||||
|
|
||||||
|
ArrayList<String> tokenEntryF = null;
|
||||||
|
|
||||||
|
Integer MarkedContinuousCounterF = null;
|
||||||
|
|
||||||
|
Integer UnmarkedPatternCounterF = null;
|
||||||
|
|
||||||
|
ArrayList<String> strTokensIpartFormF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<String> tokenFormsF = null;
|
||||||
|
|
||||||
|
ArrayList<String> strTokenEntryGetPOSF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<Integer> intTokenEntyCountsF = null;
|
||||||
|
|
||||||
|
ArrayList<String> ITokenTagsF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<String> strTokenStemsF = null;
|
||||||
|
|
||||||
|
Integer AnotatorcounterF = null;
|
||||||
|
|
||||||
|
Integer TokensCounterF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<String> entityTokenTagsF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<String> nerEntitiesF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<String> nerEntitiesTypeF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<String> stopWordTokenF = null;
|
||||||
|
|
||||||
|
java.util.ArrayList<String> stopWordLemmaF = null;
|
||||||
|
|
||||||
|
Integer PairCounterF = null;
|
||||||
|
for (String str1 : ues_copy) {
|
||||||
|
if (strF != str1) {
|
||||||
|
Annotation annotation2 = pipelineSentimentAnnotationCache.getOrDefault(str1, null);
|
||||||
|
Annotation annotation4 = pipelineAnnotationCache.getOrDefault(str1, null);
|
||||||
|
CoreDocument coreDocument1 = coreDocumentAnnotationCache.getOrDefault(str1, null);
|
||||||
|
Annotation jmweAnnotation = jmweAnnotationCache.getOrDefault(str1, null);
|
||||||
|
if (annotation2 == null) {
|
||||||
|
createStrAnnotation(str1, stanfordCoreNLPSentiment, true);
|
||||||
|
}
|
||||||
|
if (annotation4 == null) {
|
||||||
|
createStrAnnotation(str1, stanfordCoreNLP, false);
|
||||||
|
}
|
||||||
|
if (coreDocument1 == null) {
|
||||||
|
getCoreDocumentsSuggested(stanfordCoreNLP, str1);
|
||||||
|
}
|
||||||
|
if (jmweAnnotation == null) {
|
||||||
|
getJMWEAnnotation(str1);
|
||||||
|
jmweAnnotation = jmweAnnotationCache.get(str1);
|
||||||
|
}
|
||||||
|
Integer tokenizeCounting = tokenizeCountingHashMap.getOrDefault(str1, null);
|
||||||
|
|
||||||
|
List<List<TaggedWord>> taggedWordList1 = taggedWordListHashMap.getOrDefault(str1, null);
|
||||||
|
|
||||||
|
java.util.ArrayList<String> retrieveTGWList1 = retrieveTGWListHashMap.getOrDefault(str1, null);
|
||||||
|
|
||||||
|
List<CoreMap> sentence1 = sentences1HashMap.getOrDefault(str1, null);
|
||||||
|
|
||||||
|
List<CoreMap> sentenceSentiment1 = sentencesSentimentHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<Tree> trees1 = trees1HashMap.getOrDefault(str1, null);
|
||||||
|
List<CoreMap> coreMaps2 = new ArrayList<>();
|
||||||
|
ArrayList<GrammaticalStructure> grammaticalStructures1 = grammaticalStructureHashMap.getOrDefault(str1, null);
|
||||||
|
if (jmweAnnotation != null) {
|
||||||
|
coreMaps2 = jmweAnnotation.get(CoreAnnotations.SentencesAnnotation.class);
|
||||||
|
}
|
||||||
|
ArrayList<TypedDependency> typedDependencies1 = typedDependenciesHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<Integer> rnnCoreAnnotationsPredicted1 = rnnCoreAnnotationsPredictedHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<SimpleMatrix> simpleMatrices1 = simpleMatricesHashMap.getOrDefault(str1, null);
|
||||||
|
simpleMatricesHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<SimpleMatrix> simpleMatricesNodevectors1 = simpleMatricesNodevectorsHashMap.getOrDefault(str1, null);
|
||||||
|
List list1 = listHashMap.getOrDefault(str1, null);
|
||||||
|
Integer longest1 = longestHashMap.getOrDefault(str1, null);
|
||||||
|
Integer sentimentLongest1 = sentimentHashMap.getOrDefault(str1, null);
|
||||||
|
List<IMWE<IToken>> imwes1 = imwesHashMap.getOrDefault(str1, null);
|
||||||
|
Integer InflectedCounterNegative1 = InflectedCounterNegativeHashMap.getOrDefault(str1, null);
|
||||||
|
Integer InflectedCounterPositive1 = InflectedCounterPositiveHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> tokenEntry1 = tokenEntryHashMap.getOrDefault(str1, null);
|
||||||
|
Integer MarkedContinuousCounter1 = MarkedContinuousCounterHashMap.getOrDefault(str1, null);
|
||||||
|
Integer UnmarkedPatternCounter1 = UnmarkedPatternCounterHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> strTokensIpartForm1 = strTokensIpartFormHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> tokenForms1 = tokenFormsHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> strTokenEntryGetPOS1 = strTokenEntryGetPOSHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<Integer> intTokenEntyCounts1 = intTokenEntyCountsHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> ITokenTags1 = ITokenTagsHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> strTokenStems1 = strTokenStemsHashMap.getOrDefault(str1, null);
|
||||||
|
Integer Anotatorcounter1 = AnotatorcounterHashMap.getOrDefault(str1, null);
|
||||||
|
Integer TokensCounter1 = TokensCounterHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> entityTokenTags1 = entityTokenTagsHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> nerEntities1 = nerEntitiesHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> nerEntitiesType1 = nerEntitiesTypeHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> stopWordToken1 = stopWordTokenHashMap.getOrDefault(str1, null);
|
||||||
|
ArrayList<String> stopWordLemma1 = stopWordLemmaHashMap.getOrDefault(str1, null);
|
||||||
|
Integer PairCounter1 = PairCounterHashMap.getOrDefault(str1, null);
|
||||||
|
|
||||||
|
SentimentAnalyzerTest SMX = new SentimentAnalyzerTest(strF, str1, new SimilarityMatrix(strF, str1),
|
||||||
|
coreMaps1, coreMaps2, strAnno,
|
||||||
|
pipelineAnnotationCache.get(str1), strAnnoSentiment,
|
||||||
|
pipelineSentimentAnnotationCache.get(str1), coreDocument, coreDocumentAnnotationCache.get(str1),
|
||||||
|
tagger, gsf, classifier, tokenizeCounting, tokenizeCountingF,
|
||||||
|
taggedWordListF, taggedWordList1, retrieveTGWListF, retrieveTGWList1,
|
||||||
|
sentencesF, sentence1, sentencesSentimentF, sentenceSentiment1, treesF, trees1,
|
||||||
|
grammaticalStructuresF, grammaticalStructures1, typedDependenciesF,
|
||||||
|
typedDependencies1, rnnCoreAnnotationsPredictedF, rnnCoreAnnotationsPredicted1,
|
||||||
|
simpleMatricesF, simpleMatrices1, simpleMatricesNodevectorsF, simpleMatricesNodevectors1,
|
||||||
|
listF, list1, longestF, longest1, sentimentLongestF, sentimentLongest1, imwesF,
|
||||||
|
imwes1, InflectedCounterNegativeF, InflectedCounterNegative1, InflectedCounterPositiveF,
|
||||||
|
InflectedCounterPositive1, tokenEntryF, tokenEntry1, MarkedContinuousCounterF,
|
||||||
|
MarkedContinuousCounter1, UnmarkedPatternCounterF, UnmarkedPatternCounter1,
|
||||||
|
strTokensIpartFormF, strTokensIpartForm1, tokenFormsF, tokenForms1,
|
||||||
|
strTokenEntryGetPOSF, strTokenEntryGetPOS1, intTokenEntyCountsF,
|
||||||
|
intTokenEntyCounts1, ITokenTagsF, ITokenTags1, strTokenStemsF, strTokenStems1,
|
||||||
|
AnotatorcounterF, Anotatorcounter1, TokensCounterF, TokensCounter1,
|
||||||
|
entityTokenTagsF, entityTokenTags1, nerEntitiesF, nerEntities1, nerEntitiesTypeF,
|
||||||
|
nerEntitiesType1, stopWordTokenF, stopWordToken1, stopWordLemmaF, stopWordLemma1,
|
||||||
|
PairCounterF, PairCounter1
|
||||||
|
);
|
||||||
|
if (tokenizeCounting == null) {
|
||||||
|
tokenizeCountingHashMap.put(str1, SMX.getTokenizeCounting());
|
||||||
|
}
|
||||||
|
if (taggedWordList1 == null) {
|
||||||
|
taggedWordListHashMap.put(str1, SMX.getTaggedWordList1());
|
||||||
|
}
|
||||||
|
if (tokenizeCountingF == null) {
|
||||||
|
tokenizeCountingF = SMX.getTokenizeCountingF();
|
||||||
|
}
|
||||||
|
if (taggedWordListF == null) {
|
||||||
|
taggedWordListF = SMX.getTaggedWordListF();
|
||||||
|
}
|
||||||
|
if (retrieveTGWListF == null) {
|
||||||
|
retrieveTGWListF = SMX.getRetrieveTGWListF();
|
||||||
|
}
|
||||||
|
if (retrieveTGWList1 == null) {
|
||||||
|
retrieveTGWListHashMap.put(str1, SMX.getRetrieveTGWList1());
|
||||||
|
}
|
||||||
|
if (sentencesF == null) {
|
||||||
|
sentencesF = SMX.getSentencesF();
|
||||||
|
}
|
||||||
|
if (sentence1 == null) {
|
||||||
|
sentences1HashMap.put(str1, SMX.getSentences1());
|
||||||
|
}
|
||||||
|
if (sentencesSentimentF == null) {
|
||||||
|
sentencesSentimentF = SMX.getSentencesSentimentF();
|
||||||
|
}
|
||||||
|
if (sentenceSentiment1 == null) {
|
||||||
|
sentencesSentimentHashMap.put(str1, SMX.getSentencesSentiment1());
|
||||||
|
}
|
||||||
|
if (treesF == null) {
|
||||||
|
treesF = SMX.getTreesF();
|
||||||
|
}
|
||||||
|
if (trees1 == null) {
|
||||||
|
trees1HashMap.put(str1, SMX.getTrees1());
|
||||||
|
}
|
||||||
|
if (grammaticalStructuresF == null) {
|
||||||
|
grammaticalStructuresF = SMX.getGrammaticalStructuresF();
|
||||||
|
}
|
||||||
|
if (grammaticalStructures1 == null) {
|
||||||
|
grammaticalStructureHashMap.put(str1, SMX.getGrammaticalStructures1());
|
||||||
|
}
|
||||||
|
if (typedDependenciesF == null) {
|
||||||
|
typedDependenciesF = SMX.getTypedDependenciesF();
|
||||||
|
}
|
||||||
|
if (typedDependencies1 == null) {
|
||||||
|
typedDependenciesHashMap.put(str1, SMX.getTypedDependencies1());
|
||||||
|
}
|
||||||
|
if (rnnCoreAnnotationsPredictedF == null) {
|
||||||
|
rnnCoreAnnotationsPredictedF = SMX.getRnnCoreAnnotationsPredictedF();
|
||||||
|
}
|
||||||
|
if (rnnCoreAnnotationsPredicted1 == null) {
|
||||||
|
rnnCoreAnnotationsPredictedHashMap.put(str1, SMX.getRnnCoreAnnotationsPredicted1());
|
||||||
|
}
|
||||||
|
if (simpleMatricesF == null) {
|
||||||
|
simpleMatricesF = SMX.getSimpleMatricesF();
|
||||||
|
}
|
||||||
|
if (simpleMatrices1 == null) {
|
||||||
|
simpleMatricesHashMap.put(str1, SMX.getSimpleMatrices1());
|
||||||
|
}
|
||||||
|
if (simpleMatricesNodevectorsF == null) {
|
||||||
|
simpleMatricesNodevectorsF = SMX.getSimpleMatricesNodevectorsF();
|
||||||
|
}
|
||||||
|
if (simpleMatricesNodevectors1 == null) {
|
||||||
|
simpleMatricesNodevectorsHashMap.put(str1, SMX.getSimpleMatricesNodevectors1());
|
||||||
|
}
|
||||||
|
if (listF == null) {
|
||||||
|
listF = SMX.getListF();
|
||||||
|
}
|
||||||
|
if (list1 == null) {
|
||||||
|
listHashMap.put(str1, SMX.getList1());
|
||||||
|
}
|
||||||
|
if (longestF == null) {
|
||||||
|
longestF = SMX.getLongestF();
|
||||||
|
}
|
||||||
|
if (longest1 == null) {
|
||||||
|
longestHashMap.put(str1, SMX.getLongest1());
|
||||||
|
}
|
||||||
|
if (sentimentLongestF == null) {
|
||||||
|
sentimentLongestF = SMX.getSentimentLongestF();
|
||||||
|
}
|
||||||
|
if (sentimentLongest1 == null) {
|
||||||
|
sentimentHashMap.put(str1, SMX.getSentimentLongest1());
|
||||||
|
}
|
||||||
|
if (imwesF == null) {
|
||||||
|
imwesF = SMX.getImwesF();
|
||||||
|
}
|
||||||
|
if (imwes1 == null) {
|
||||||
|
imwesHashMap.put(str1, SMX.getImwes1());
|
||||||
|
}
|
||||||
|
if (InflectedCounterNegativeF == null) {
|
||||||
|
InflectedCounterNegativeF = SMX.getInflectedCounterNegativeF();
|
||||||
|
}
|
||||||
|
if (InflectedCounterNegative1 == null) {
|
||||||
|
InflectedCounterNegativeHashMap.put(str1, SMX.getInflectedCounterNegative1());
|
||||||
|
}
|
||||||
|
if (InflectedCounterPositiveF == null) {
|
||||||
|
InflectedCounterPositiveF = SMX.getInflectedCounterPositiveF();
|
||||||
|
}
|
||||||
|
if (InflectedCounterPositive1 == null) {
|
||||||
|
InflectedCounterPositiveHashMap.put(str1, SMX.getInflectedCounterPositive1());
|
||||||
|
}
|
||||||
|
if (tokenEntryF == null) {
|
||||||
|
tokenEntryF = SMX.getTokenEntryF();
|
||||||
|
}
|
||||||
|
if (tokenEntry1 == null) {
|
||||||
|
tokenEntryHashMap.put(str1, SMX.getTokenEntry1());
|
||||||
|
}
|
||||||
|
if (MarkedContinuousCounterF == null) {
|
||||||
|
MarkedContinuousCounterF = SMX.getMarkedContinuousCounterF();
|
||||||
|
}
|
||||||
|
if (MarkedContinuousCounter1 == null) {
|
||||||
|
MarkedContinuousCounterHashMap.put(str1, SMX.getMarkedContinuousCounter1());
|
||||||
|
}
|
||||||
|
if (UnmarkedPatternCounterF == null) {
|
||||||
|
UnmarkedPatternCounterF = SMX.getUnmarkedPatternCounterF();
|
||||||
|
}
|
||||||
|
if (UnmarkedPatternCounter1 == null) {
|
||||||
|
UnmarkedPatternCounterHashMap.put(str1, SMX.getUnmarkedPatternCounter1());
|
||||||
|
}
|
||||||
|
if (strTokensIpartFormF == null) {
|
||||||
|
strTokensIpartFormF = SMX.getStrTokensIpartFormF();
|
||||||
|
}
|
||||||
|
if (strTokensIpartForm1 == null) {
|
||||||
|
strTokensIpartFormHashMap.put(str1, SMX.getStrTokensIpartForm1());
|
||||||
|
}
|
||||||
|
if (tokenFormsF == null) {
|
||||||
|
tokenFormsF = SMX.getTokenFormsF();
|
||||||
|
}
|
||||||
|
if (tokenForms1 == null) {
|
||||||
|
tokenFormsHashMap.put(str1, SMX.getTokenForms1());
|
||||||
|
}
|
||||||
|
if (strTokenEntryGetPOSF == null) {
|
||||||
|
strTokenEntryGetPOSF = SMX.getStrTokenEntryGetPOSF();
|
||||||
|
}
|
||||||
|
if (strTokenEntryGetPOS1 == null) {
|
||||||
|
strTokenEntryGetPOSHashMap.put(str1, SMX.getStrTokenEntryGetPOS1());
|
||||||
|
}
|
||||||
|
if (intTokenEntyCountsF == null) {
|
||||||
|
intTokenEntyCountsF = SMX.getIntTokenEntyCountsF();
|
||||||
|
}
|
||||||
|
if (intTokenEntyCounts1 == null) {
|
||||||
|
intTokenEntyCountsHashMap.put(str1, SMX.getIntTokenEntyCounts1());
|
||||||
|
}
|
||||||
|
if (ITokenTagsF == null) {
|
||||||
|
ITokenTagsF = SMX.getITokenTagsF();
|
||||||
|
}
|
||||||
|
if (ITokenTags1 == null) {
|
||||||
|
ITokenTagsHashMap.put(str1, SMX.getITokenTags1());
|
||||||
|
}
|
||||||
|
if (strTokenStemsF == null) {
|
||||||
|
strTokenStemsF = SMX.getStrTokenStemsF();
|
||||||
|
}
|
||||||
|
if (strTokenStems1 == null) {
|
||||||
|
strTokenStemsHashMap.put(str1, SMX.getStrTokenStems1());
|
||||||
|
}
|
||||||
|
if (AnotatorcounterF == null) {
|
||||||
|
AnotatorcounterF = SMX.getAnotatorcounterF();
|
||||||
|
}
|
||||||
|
if (Anotatorcounter1 == null) {
|
||||||
|
AnotatorcounterHashMap.put(str1, SMX.getAnotatorcounter1());
|
||||||
|
}
|
||||||
|
if (TokensCounterF == null) {
|
||||||
|
TokensCounterF = SMX.getTokensCounterF();
|
||||||
|
}
|
||||||
|
if (TokensCounter1 == null) {
|
||||||
|
TokensCounterHashMap.put(str1, SMX.getTokensCounter1());
|
||||||
|
}
|
||||||
|
if (entityTokenTagsF == null) {
|
||||||
|
entityTokenTagsF = SMX.getEntityTokenTagsF();
|
||||||
|
}
|
||||||
|
if (entityTokenTags1 == null) {
|
||||||
|
entityTokenTagsHashMap.put(str1, SMX.getEntityTokenTags1());
|
||||||
|
}
|
||||||
|
if (nerEntitiesF == null) {
|
||||||
|
nerEntitiesF = SMX.getNerEntitiesF();
|
||||||
|
}
|
||||||
|
if (nerEntities1 == null) {
|
||||||
|
nerEntitiesHashMap.put(str1, SMX.getNerEntities1());
|
||||||
|
}
|
||||||
|
if (nerEntitiesTypeF == null) {
|
||||||
|
nerEntitiesTypeF = SMX.getNerEntitiesTypeF();
|
||||||
|
}
|
||||||
|
if (nerEntitiesType1 == null) {
|
||||||
|
nerEntitiesTypeHashMap.put(str1, SMX.getNerEntitiesType1());
|
||||||
|
}
|
||||||
|
if (stopWordTokenF == null) {
|
||||||
|
stopWordTokenF = SMX.getStopWordTokenF();
|
||||||
|
}
|
||||||
|
if (stopWordToken1 == null) {
|
||||||
|
stopWordTokenHashMap.put(str1, SMX.getStopWordToken1());
|
||||||
|
}
|
||||||
|
if (stopWordLemmaF == null) {
|
||||||
|
stopWordLemmaF = SMX.getStopWordLemmaF();
|
||||||
|
}
|
||||||
|
if (stopWordLemma1 == null) {
|
||||||
|
stopWordLemmaHashMap.put(str1, SMX.getStopWordLemma1());
|
||||||
|
}
|
||||||
|
if (PairCounterF == null) {
|
||||||
|
PairCounterF = SMX.getPairCounterF();
|
||||||
|
}
|
||||||
|
if (PairCounter1 == null) {
|
||||||
|
PairCounterHashMap.put(str1, SMX.getPairCounter1());
|
||||||
|
}
|
||||||
|
|
||||||
|
SimilarityMatrix getSMX = SMX.callSMX();
|
||||||
|
double scoreRelationLastUserMsg = getSMX.getDistance();
|
||||||
|
if (scoreRelationLastUserMsg > preRelationUserCounters) {
|
||||||
|
preRelationUserCounters = scoreRelationLastUserMsg;
|
||||||
|
concurrentRelations.add(getSMX.getSecondaryString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
int cacheRequirement = 6500;
|
||||||
|
if (preRelationUserCounters > cacheRequirement && !ues_copy.contains(strF) && filterContent(strF)) {
|
||||||
|
DataMapper.InsertMYSQLStrings(strF);
|
||||||
|
DataMapper.checkStringsToDelete();
|
||||||
|
}
|
||||||
|
double randomLenghtPermit = strF.length() * (Math.random() * Math.random() * Math.random() * (Math.random() * 10));
|
||||||
|
Collections.reverse(concurrentRelations);
|
||||||
|
ArrayList<String> mysqlUpdateLastUsed = new ArrayList();
|
||||||
|
if (!concurrentRelations.isEmpty()) {
|
||||||
|
for (String secondaryRelation : concurrentRelations) {
|
||||||
|
if (SB.toString().length() > randomLenghtPermit && !SB.toString().isEmpty()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
SB.append(secondaryRelation).append(" ");
|
||||||
|
mysqlUpdateLastUsed.add(secondaryRelation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (SB.toString().isEmpty()) {
|
||||||
|
return "failure, preventing stuckness";
|
||||||
|
}
|
||||||
|
DataMapper.updateLastUsed(mysqlUpdateLastUsed);
|
||||||
|
return SB.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void getJMWEAnnotation(String str1) {
|
||||||
|
Annotation jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str1);
|
||||||
|
jmweAnnotationCache.put(str1, jmweAnnotation);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getResponseMsg(String str, String personName, StanfordCoreNLP stanfordCoreNLP,
|
||||||
|
StanfordCoreNLP stanfordCoreNLPSentiment, Boolean ingameResponse) {
|
||||||
|
String responseFutures = "";
|
||||||
|
String strF = trimString(str);
|
||||||
|
responseFutures = getResponseFutures(strF, stanfordCoreNLP, stanfordCoreNLPSentiment);
|
||||||
|
if (!ingameResponse) {
|
||||||
|
responseFutures = checkPersonPresentInSentence(personName, responseFutures, strF, stanfordCoreNLP,
|
||||||
|
stanfordCoreNLPSentiment);
|
||||||
|
}
|
||||||
|
return responseFutures;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String checkPersonPresentInSentence(String personName, String responseMsg, String userLastMessage,
|
||||||
|
StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) {
|
||||||
|
try {
|
||||||
|
CoreDocument pipelineCoreDcoument = new CoreDocument(responseMsg);
|
||||||
|
CoreDocument pipelineCoreDcoumentLastMsg = new CoreDocument(userLastMessage);
|
||||||
|
stanfordCoreNLP.annotate(pipelineCoreDcoument);
|
||||||
|
stanfordCoreNLPSentiment.annotate(pipelineCoreDcoumentLastMsg);
|
||||||
|
String regex = "(.*?\\d){10,}";
|
||||||
|
for (CoreEntityMention em : pipelineCoreDcoument.entityMentions()) {
|
||||||
|
String entityType = em.entityType();
|
||||||
|
if (entityType == "PERSON") {
|
||||||
|
String str = responseMsg;
|
||||||
|
String emText = em.text();
|
||||||
|
Pattern pattern = Pattern.compile(regex);
|
||||||
|
Matcher matcher = pattern.matcher(personName);
|
||||||
|
boolean isMatched = matcher.matches();
|
||||||
|
if (emText != personName && !isMatched) {
|
||||||
|
for (CoreEntityMention emLastMsg : pipelineCoreDcoumentLastMsg.entityMentions()) {
|
||||||
|
if (emText != emLastMsg.text() && !Character.isDigit(Integer.parseInt(emLastMsg.text().trim()))) {
|
||||||
|
str = (responseMsg.substring(0, responseMsg.indexOf(emText)) + " "
|
||||||
|
+ emLastMsg + " " + responseMsg.substring(responseMsg.indexOf(emText)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
str += personName;
|
||||||
|
return str;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println("SCUFFED JAYZ: " + e.getMessage());
|
||||||
|
}
|
||||||
|
return responseMsg;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean filterContent(String str) {
|
||||||
|
if (!str.isEmpty() && str.length() > 3) {
|
||||||
|
String str1Local = str.trim();
|
||||||
|
if (str1Local.length() > 2 && !str1Local.startsWith("!")) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void getCoreDocumentsSuggested(StanfordCoreNLP pipeline, String str) {
|
||||||
|
Annotation annotation = new Annotation(str);
|
||||||
|
pipeline.annotate(annotation);
|
||||||
|
CoreDocument coreDocument = new CoreDocument(annotation);
|
||||||
|
coreDocumentAnnotationCache.put(str, coreDocument);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,628 @@
|
|||||||
|
/*
|
||||||
|
* To change this license header, choose License Headers in Project Properties.
|
||||||
|
* To change this template file, choose Tools | Templates
|
||||||
|
* and open the template in the editor.
|
||||||
|
*/
|
||||||
|
package FunctionLayer
|
||||||
|
|
||||||
|
import DataLayer.DataMapper
|
||||||
|
import FunctionLayer.StanfordParser.SentimentAnalyzerTest
|
||||||
|
import edu.mit.jmwe.data.IMWE
|
||||||
|
import edu.mit.jmwe.data.IToken
|
||||||
|
import edu.stanford.nlp.ie.AbstractSequenceClassifier
|
||||||
|
import edu.stanford.nlp.ie.crf.CRFClassifier
|
||||||
|
import edu.stanford.nlp.ling.CoreAnnotations
|
||||||
|
import edu.stanford.nlp.ling.CoreLabel
|
||||||
|
import edu.stanford.nlp.ling.TaggedWord
|
||||||
|
import edu.stanford.nlp.parser.lexparser.LexicalizedParser
|
||||||
|
import edu.stanford.nlp.pipeline.Annotation
|
||||||
|
import edu.stanford.nlp.pipeline.CoreDocument
|
||||||
|
import edu.stanford.nlp.pipeline.StanfordCoreNLP
|
||||||
|
import edu.stanford.nlp.tagger.maxent.MaxentTagger
|
||||||
|
import edu.stanford.nlp.trees.*
|
||||||
|
import edu.stanford.nlp.util.CoreMap
|
||||||
|
import kotlinx.coroutines.*
|
||||||
|
import org.ejml.simple.SimpleMatrix
|
||||||
|
import java.util.*
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
import java.util.regex.Pattern
|
||||||
|
import kotlin.collections.ArrayList
|
||||||
|
import kotlin.collections.HashMap
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @author install1
|
||||||
|
*/
|
||||||
|
public class DatahandlerKotlinObsolete {
|
||||||
|
private var pipelineAnnotationCache: HashMap<String, Annotation>
|
||||||
|
private var pipelineSentimentAnnotationCache = HashMap<String, Annotation>()
|
||||||
|
private var coreDocumentAnnotationCache: HashMap<String, CoreDocument>
|
||||||
|
private var jmweAnnotationCache = HashMap<String, Annotation>()
|
||||||
|
|
||||||
|
//private val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz"
|
||||||
|
private val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz"
|
||||||
|
private var tagger: MaxentTagger = MaxentTagger()
|
||||||
|
private var gsf: GrammaticalStructureFactory
|
||||||
|
private var classifier: AbstractSequenceClassifier<CoreLabel>
|
||||||
|
|
||||||
|
//SentimentAnalyzer Hashmaps
|
||||||
|
private var tokenizeCountingHashMap: HashMap<String, Int> = HashMap()
|
||||||
|
private var taggedWordListHashMap: HashMap<String, List<List<TaggedWord>>> = HashMap()
|
||||||
|
private var retrieveTGWListHashMap: HashMap<String, java.util.ArrayList<String>> =
|
||||||
|
HashMap()
|
||||||
|
private var sentences1HashMap: HashMap<String, List<CoreMap>> = HashMap()
|
||||||
|
private var sentencesSentimentHashMap: HashMap<String, List<CoreMap>> = HashMap()
|
||||||
|
private var trees1HashMap: HashMap<String, java.util.ArrayList<Tree>> = HashMap()
|
||||||
|
private var grammaticalStructureHashMap: HashMap<String, java.util.ArrayList<GrammaticalStructure>> =
|
||||||
|
HashMap()
|
||||||
|
private var typedDependenciesHashMap: HashMap<String, java.util.ArrayList<TypedDependency>> =
|
||||||
|
HashMap()
|
||||||
|
private var rnnCoreAnnotationsPredictedHashMap: HashMap<String, java.util.ArrayList<Int>> = HashMap()
|
||||||
|
private var simpleMatricesHashMap: HashMap<String, java.util.ArrayList<SimpleMatrix>> = HashMap()
|
||||||
|
private var simpleMatricesNodevectorsHashMap: HashMap<String, java.util.ArrayList<SimpleMatrix>> = HashMap()
|
||||||
|
private var listHashMap: HashMap<String, MutableList<Any?>> = HashMap()
|
||||||
|
private var longestHashMap: HashMap<String, Int> = HashMap()
|
||||||
|
private var sentimentHashMap: HashMap<String, Int> = HashMap()
|
||||||
|
private var imwesHashMap: HashMap<String, List<IMWE<IToken>>> = HashMap()
|
||||||
|
private var InflectedCounterNegativeHashMap: HashMap<String, Int> = HashMap()
|
||||||
|
private var InflectedCounterPositiveHashMap: HashMap<String, Int> = HashMap()
|
||||||
|
private var tokenEntryHashMap: HashMap<String, ArrayList<String>> = HashMap()
|
||||||
|
private var MarkedContinuousCounterHashMap: HashMap<String, Int> = HashMap()
|
||||||
|
private var UnmarkedPatternCounterHashMap: HashMap<String, Int> = HashMap()
|
||||||
|
private var strTokensIpartFormHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||||
|
private var tokenFormsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||||
|
private var strTokenEntryGetPOSHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||||
|
private var intTokenEntyCountsHashMap: HashMap<String, java.util.ArrayList<Int>> = HashMap()
|
||||||
|
private var ITokenTagsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||||
|
private var strTokenStemsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||||
|
private var AnotatorcounterHashMap: HashMap<String, Int> = HashMap()
|
||||||
|
private var TokensCounterHashMap: HashMap<String, Int> = HashMap()
|
||||||
|
private var entityTokenTagsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||||
|
private var nerEntitiesHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||||
|
private var nerEntitiesTypeHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||||
|
private var stopWordTokenHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||||
|
private var stopWordLemmaHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||||
|
private var PairCounterHashMap: HashMap<String, Int> = HashMap()
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
jmweAnnotationCache = HashMap<String, Annotation>()
|
||||||
|
pipelineAnnotationCache = HashMap<String, Annotation>()
|
||||||
|
pipelineSentimentAnnotationCache = HashMap<String, Annotation>()
|
||||||
|
coreDocumentAnnotationCache = HashMap<String, CoreDocument>()
|
||||||
|
gsf = initiateGrammaticalStructureFactory()
|
||||||
|
classifier = CRFClassifier.getClassifierNoExceptions(nerModel)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun initiateGrammaticalStructureFactory(): GrammaticalStructureFactory {
|
||||||
|
val options = arrayOf("-maxLength", "100")
|
||||||
|
//val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
|
||||||
|
val lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"
|
||||||
|
val lp = LexicalizedParser.loadModel(lexParserEnglishPCFG, *options)
|
||||||
|
val tlp = lp.getOp().langpack()
|
||||||
|
return tlp.grammaticalStructureFactory()
|
||||||
|
}
|
||||||
|
|
||||||
|
public fun pipeLineSetUp(): StanfordCoreNLP {
|
||||||
|
val props = Properties()
|
||||||
|
val shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz"
|
||||||
|
//val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz"
|
||||||
|
val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz"
|
||||||
|
//val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz"
|
||||||
|
val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.distsim.crf.ser.gz"
|
||||||
|
props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse")
|
||||||
|
props.setProperty("parse.model", shiftReduceParserPath)
|
||||||
|
props.setProperty("parse.maxlen", "90")
|
||||||
|
props.setProperty("parse.binaryTrees", "true")
|
||||||
|
props.setProperty("threads", "5")
|
||||||
|
props.setProperty("pos.maxlen", "90")
|
||||||
|
props.setProperty("tokenize.maxlen", "90")
|
||||||
|
props.setProperty("ssplit.maxlen", "90")
|
||||||
|
props.setProperty("lemma.maxlen", "90")
|
||||||
|
props.setProperty("ner.model", "$nerModel,$nerModel2,$nerModel3")
|
||||||
|
props.setProperty("ner.combinationMode", "HIGH_RECALL")
|
||||||
|
props.setProperty("regexner.ignorecase", "true")
|
||||||
|
props.setProperty("ner.fine.regexner.ignorecase", "true")
|
||||||
|
props.setProperty("tokenize.options", "untokenizable=firstKeep")
|
||||||
|
return StanfordCoreNLP(props)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun shiftReduceParserInitiate(): StanfordCoreNLP {
|
||||||
|
val propsSentiment = Properties()
|
||||||
|
//val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
|
||||||
|
val lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"
|
||||||
|
val sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz"
|
||||||
|
//val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger"
|
||||||
|
val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words-distsim.tagger"
|
||||||
|
val customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of," +
|
||||||
|
"on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"
|
||||||
|
propsSentiment.setProperty("parse.model", lexParserEnglishPCFG)
|
||||||
|
propsSentiment.setProperty("sentiment.model", sentimentModel)
|
||||||
|
propsSentiment.setProperty("parse.maxlen", "90")
|
||||||
|
propsSentiment.setProperty("threads", "5")
|
||||||
|
propsSentiment.setProperty("pos.maxlen", "90")
|
||||||
|
propsSentiment.setProperty("tokenize.maxlen", "90")
|
||||||
|
propsSentiment.setProperty("ssplit.maxlen", "90")
|
||||||
|
propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword") //coref too expensive memorywise
|
||||||
|
propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator")
|
||||||
|
propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList)
|
||||||
|
propsSentiment.setProperty("tokenize.options", "untokenizable=firstKeep")
|
||||||
|
tagger = MaxentTagger(taggerPath)
|
||||||
|
|
||||||
|
println("finished shiftReduceParserInitiate\n")
|
||||||
|
return StanfordCoreNLP(propsSentiment)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun trimString(str: String): String {
|
||||||
|
var message = str.trim { it <= ' ' }
|
||||||
|
if (message.startsWith("<@")) {
|
||||||
|
message = message.substring(message.indexOf("> ") + 2)
|
||||||
|
}
|
||||||
|
if (!message.isEmpty()) {
|
||||||
|
message = message.replace("@", "")
|
||||||
|
if (message.contains("<>")) {
|
||||||
|
message = message.substring(message.indexOf(">"))
|
||||||
|
}
|
||||||
|
if (message.startsWith("[ *")) {
|
||||||
|
message = message.substring(message.indexOf("]"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return message
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun createStrAnnotation(str: String, stanfordCoreNLP: StanfordCoreNLP, sentimentBool: Boolean) {
|
||||||
|
val strAnno2 = Annotation(str)
|
||||||
|
strAnno2.compact()
|
||||||
|
stanfordCoreNLP.annotate(strAnno2)
|
||||||
|
if (sentimentBool) {
|
||||||
|
pipelineSentimentAnnotationCache.put(str, strAnno2)
|
||||||
|
} else {
|
||||||
|
pipelineAnnotationCache.put(str, strAnno2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun getResponseFutures(strF: String, stanfordCoreNLP: StanfordCoreNLP, stanfordCoreNLPSentiment: StanfordCoreNLP): String {
|
||||||
|
val strAnno: Annotation = Annotation(strF)
|
||||||
|
strAnno.compact()
|
||||||
|
stanfordCoreNLP.annotate(strAnno)
|
||||||
|
|
||||||
|
val strAnnoSentiment: Annotation = Annotation(strF)
|
||||||
|
strAnnoSentiment.compact()
|
||||||
|
stanfordCoreNLPSentiment.annotate(strAnnoSentiment)
|
||||||
|
|
||||||
|
|
||||||
|
val annotation = Annotation(strF)
|
||||||
|
stanfordCoreNLP.annotate(annotation)
|
||||||
|
val coreDocument = CoreDocument(annotation)
|
||||||
|
|
||||||
|
val values_copy: List<String> = ArrayList(DataMapper.getAllStrings())
|
||||||
|
var preRelationUserCounters = -155000.0
|
||||||
|
val concurrentRelations: MutableList<String> = arrayListOf()
|
||||||
|
val SB = StringBuilder()
|
||||||
|
var jmweAnnotationF = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strF)
|
||||||
|
var tokenizeCountingF: Int? = null
|
||||||
|
var taggedWordListF: List<List<TaggedWord>>? = null
|
||||||
|
var retrieveTGWListF: java.util.ArrayList<String>? = null
|
||||||
|
var sentencesF: List<CoreMap>? = null
|
||||||
|
var sentencesSentimentF: List<CoreMap>? = null
|
||||||
|
var coreMaps1: List<CoreMap> = jmweAnnotationF.get(CoreAnnotations.SentencesAnnotation::class.java)
|
||||||
|
var treesF: java.util.ArrayList<Tree>? = null
|
||||||
|
var grammaticalStructuresF: ArrayList<GrammaticalStructure>? = null
|
||||||
|
var typedDependenciesF: java.util.ArrayList<TypedDependency>? = null
|
||||||
|
var rnnCoreAnnotationsPredictedF: java.util.ArrayList<Int>? = null
|
||||||
|
var simpleMatricesF: java.util.ArrayList<SimpleMatrix>? = null
|
||||||
|
var simpleMatricesNodevectorsF: java.util.ArrayList<SimpleMatrix>? = null
|
||||||
|
var listF: MutableList<Any?>? = null
|
||||||
|
var longestF: Int? = null
|
||||||
|
var sentimentLongestF: Int? = null
|
||||||
|
var imwesF: List<IMWE<IToken>>? = null
|
||||||
|
var InflectedCounterNegativeF: Int? = null
|
||||||
|
var InflectedCounterPositiveF: Int? = null
|
||||||
|
var tokenEntryF: ArrayList<String>? = null
|
||||||
|
var MarkedContinuousCounterF: Int? = null
|
||||||
|
var UnmarkedPatternCounterF: Int? = null
|
||||||
|
var strTokensIpartFormF: ArrayList<String>? = null
|
||||||
|
var tokenFormsF: java.util.ArrayList<String>? = null
|
||||||
|
var strTokenEntryGetPOSF: ArrayList<String>? = null
|
||||||
|
var intTokenEntyCountsF: java.util.ArrayList<Int>? = null
|
||||||
|
var ITokenTagsF: ArrayList<String>? = null
|
||||||
|
var strTokenStemsF: java.util.ArrayList<String>? = null
|
||||||
|
var AnotatorcounterF: Int? = null
|
||||||
|
var TokensCounterF: Int? = null
|
||||||
|
var entityTokenTagsF: java.util.ArrayList<String>? = null
|
||||||
|
var nerEntitiesF: java.util.ArrayList<String>? = null
|
||||||
|
var nerEntitiesTypeF: java.util.ArrayList<String>? = null
|
||||||
|
var stopWordTokenF: java.util.ArrayList<String>? = null
|
||||||
|
var stopWordLemmaF: java.util.ArrayList<String>? = null
|
||||||
|
var PairCounterF: Int? = null
|
||||||
|
for (str1 in values_copy) {
|
||||||
|
if (strF != str1) {
|
||||||
|
val annotation2 = pipelineSentimentAnnotationCache.getOrDefault(str1, null)
|
||||||
|
val annotation4 = pipelineAnnotationCache.getOrDefault(str1, null)
|
||||||
|
val coreDocument1 = coreDocumentAnnotationCache.getOrDefault(str1, null)
|
||||||
|
var jmweAnnotation = jmweAnnotationCache.getOrDefault(str1, null)
|
||||||
|
if (annotation2 == null) {
|
||||||
|
createStrAnnotation(str1, stanfordCoreNLPSentiment, true)
|
||||||
|
}
|
||||||
|
if (annotation4 == null) {
|
||||||
|
createStrAnnotation(str1, stanfordCoreNLP, false)
|
||||||
|
}
|
||||||
|
if (coreDocument1 == null) {
|
||||||
|
getCoreDocumentsSuggested(stanfordCoreNLP, str1)
|
||||||
|
}
|
||||||
|
if (jmweAnnotation == null) {
|
||||||
|
getJMWEAnnotation(str1)
|
||||||
|
jmweAnnotation = jmweAnnotationCache.get(str1)
|
||||||
|
}
|
||||||
|
val tokenizeCounting: Int? = tokenizeCountingHashMap.getOrDefault(str1, null)
|
||||||
|
val taggedWordList1: List<List<TaggedWord>>? = taggedWordListHashMap.getOrDefault(str1, null)
|
||||||
|
val retrieveTGWList1: java.util.ArrayList<String>? = retrieveTGWListHashMap.getOrDefault(str1, null)
|
||||||
|
val sentence1: List<CoreMap>? = sentences1HashMap.getOrDefault(str1, null)
|
||||||
|
val sentenceSentiment1: List<CoreMap>? = sentencesSentimentHashMap.getOrDefault(str1, null)
|
||||||
|
val trees1 = trees1HashMap.getOrDefault(str1, null)
|
||||||
|
var coreMaps2: List<CoreMap> = listOf()
|
||||||
|
val grammaticalStructures1 = grammaticalStructureHashMap.getOrDefault(
|
||||||
|
str1, null)
|
||||||
|
if (jmweAnnotation != null) {
|
||||||
|
coreMaps2 = jmweAnnotation.get(CoreAnnotations.SentencesAnnotation::class.java)
|
||||||
|
}
|
||||||
|
val typedDependencies1 = typedDependenciesHashMap.getOrDefault(str1, null)
|
||||||
|
val rnnCoreAnnotationsPredicted1 = rnnCoreAnnotationsPredictedHashMap.getOrDefault(str1, null)
|
||||||
|
val simpleMatrices1 = simpleMatricesHashMap.getOrDefault(str1, null);
|
||||||
|
val simpleMatricesNodevectors1 = simpleMatricesNodevectorsHashMap.getOrDefault(str1, null);
|
||||||
|
val list1 = listHashMap.getOrDefault(str1, null);
|
||||||
|
val longest1 = longestHashMap.getOrDefault(str1, null);
|
||||||
|
val sentimentLongest1 = sentimentHashMap.getOrDefault(str1, null);
|
||||||
|
val imwes1 = imwesHashMap.getOrDefault(str1, null);
|
||||||
|
val InflectedCounterNegative1 = InflectedCounterNegativeHashMap.getOrDefault(str1, null);
|
||||||
|
val InflectedCounterPositive1 = InflectedCounterPositiveHashMap.getOrDefault(str1, null)
|
||||||
|
val tokenEntry1 = tokenEntryHashMap.getOrDefault(str1, null)
|
||||||
|
val MarkedContinuousCounter1 = MarkedContinuousCounterHashMap.getOrDefault(str1, null)
|
||||||
|
val UnmarkedPatternCounter1 = UnmarkedPatternCounterHashMap.getOrDefault(str1, null)
|
||||||
|
val strTokensIpartForm1 = strTokensIpartFormHashMap.getOrDefault(str1, null);
|
||||||
|
val tokenForms1 = tokenFormsHashMap.getOrDefault(str1, null);
|
||||||
|
val strTokenEntryGetPOS1 = strTokenEntryGetPOSHashMap.getOrDefault(str1, null)
|
||||||
|
val intTokenEntyCounts1 = intTokenEntyCountsHashMap.getOrDefault(str1, null);
|
||||||
|
val ITokenTags1 = ITokenTagsHashMap.getOrDefault(str1, null);
|
||||||
|
val strTokenStems1 = strTokenStemsHashMap.getOrDefault(str1, null);
|
||||||
|
val Anotatorcounter1 = AnotatorcounterHashMap.getOrDefault(str1, null);
|
||||||
|
val TokensCounter1 = TokensCounterHashMap.getOrDefault(str1, null);
|
||||||
|
val entityTokenTags1 = entityTokenTagsHashMap.getOrDefault(str1, null);
|
||||||
|
val nerEntities1 = nerEntitiesHashMap.getOrDefault(str1, null);
|
||||||
|
val nerEntitiesType1 = nerEntitiesTypeHashMap.getOrDefault(str1, null);
|
||||||
|
val stopWordToken1 = stopWordTokenHashMap.getOrDefault(str1, null);
|
||||||
|
val stopWordLemma1 = stopWordLemmaHashMap.getOrDefault(str1, null);
|
||||||
|
val PairCounter1 = PairCounterHashMap.getOrDefault(str1, null);
|
||||||
|
|
||||||
|
var SMX = SentimentAnalyzerTest(strF, str1, SimilarityMatrix(strF, str1),
|
||||||
|
coreMaps1, coreMaps2, strAnno,
|
||||||
|
pipelineAnnotationCache[str1], strAnnoSentiment,
|
||||||
|
pipelineSentimentAnnotationCache[str1], coreDocument, coreDocumentAnnotationCache[str1],
|
||||||
|
tagger, gsf, classifier, tokenizeCounting, tokenizeCountingF,
|
||||||
|
taggedWordListF, taggedWordList1, retrieveTGWListF, retrieveTGWList1,
|
||||||
|
sentencesF, sentence1, sentencesSentimentF, sentenceSentiment1, treesF, trees1,
|
||||||
|
grammaticalStructuresF, grammaticalStructures1, typedDependenciesF,
|
||||||
|
typedDependencies1, rnnCoreAnnotationsPredictedF, rnnCoreAnnotationsPredicted1,
|
||||||
|
simpleMatricesF, simpleMatrices1, simpleMatricesNodevectorsF, simpleMatricesNodevectors1,
|
||||||
|
listF, list1, longestF, longest1, sentimentLongestF, sentimentLongest1, imwesF,
|
||||||
|
imwes1, InflectedCounterNegativeF, InflectedCounterNegative1, InflectedCounterPositiveF,
|
||||||
|
InflectedCounterPositive1, tokenEntryF, tokenEntry1, MarkedContinuousCounterF,
|
||||||
|
MarkedContinuousCounter1, UnmarkedPatternCounterF, UnmarkedPatternCounter1,
|
||||||
|
strTokensIpartFormF, strTokensIpartForm1, tokenFormsF, tokenForms1,
|
||||||
|
strTokenEntryGetPOSF, strTokenEntryGetPOS1, intTokenEntyCountsF,
|
||||||
|
intTokenEntyCounts1, ITokenTagsF, ITokenTags1, strTokenStemsF, strTokenStems1,
|
||||||
|
AnotatorcounterF, Anotatorcounter1, TokensCounterF, TokensCounter1,
|
||||||
|
entityTokenTagsF, entityTokenTags1, nerEntitiesF, nerEntities1, nerEntitiesTypeF,
|
||||||
|
nerEntitiesType1, stopWordTokenF, stopWordToken1, stopWordLemmaF, stopWordLemma1,
|
||||||
|
PairCounterF, PairCounter1)
|
||||||
|
if (tokenizeCounting == null) {
|
||||||
|
tokenizeCountingHashMap.put(str1, SMX.getTokenizeCounting())
|
||||||
|
}
|
||||||
|
if (taggedWordList1 == null) {
|
||||||
|
taggedWordListHashMap.put(str1, SMX.getTaggedWordList1())
|
||||||
|
}
|
||||||
|
if (tokenizeCountingF == null) {
|
||||||
|
tokenizeCountingF = SMX.getTokenizeCountingF();
|
||||||
|
}
|
||||||
|
if (taggedWordListF == null) {
|
||||||
|
taggedWordListF = SMX.getTaggedWordListF();
|
||||||
|
}
|
||||||
|
if (retrieveTGWListF == null) {
|
||||||
|
retrieveTGWListF = SMX.getRetrieveTGWListF();
|
||||||
|
}
|
||||||
|
if (retrieveTGWList1 == null) {
|
||||||
|
retrieveTGWListHashMap.put(str1, SMX.getRetrieveTGWList1());
|
||||||
|
}
|
||||||
|
if (sentencesF == null) {
|
||||||
|
sentencesF = SMX.getSentencesF();
|
||||||
|
}
|
||||||
|
if (sentence1 == null) {
|
||||||
|
sentences1HashMap.put(str1, SMX.getSentences1())
|
||||||
|
}
|
||||||
|
if (sentencesSentimentF == null) {
|
||||||
|
sentencesSentimentF = SMX.getSentencesSentimentF();
|
||||||
|
}
|
||||||
|
if (sentenceSentiment1 == null) {
|
||||||
|
sentencesSentimentHashMap.put(str1, SMX.getSentencesSentiment1());
|
||||||
|
}
|
||||||
|
if (treesF == null) {
|
||||||
|
treesF = SMX.getTreesF();
|
||||||
|
}
|
||||||
|
if (trees1 == null) {
|
||||||
|
trees1HashMap.put(str1, SMX.getTrees1())
|
||||||
|
}
|
||||||
|
if (grammaticalStructuresF == null) {
|
||||||
|
grammaticalStructuresF = SMX.getGrammaticalStructuresF();
|
||||||
|
}
|
||||||
|
if (grammaticalStructures1 == null) {
|
||||||
|
grammaticalStructureHashMap.put(str1, SMX.getGrammaticalStructures1())
|
||||||
|
}
|
||||||
|
if (typedDependenciesF == null) {
|
||||||
|
typedDependenciesF = SMX.getTypedDependenciesF();
|
||||||
|
}
|
||||||
|
if (typedDependencies1 == null) {
|
||||||
|
typedDependenciesHashMap.put(str1, SMX.getTypedDependencies1())
|
||||||
|
}
|
||||||
|
if (rnnCoreAnnotationsPredictedF == null) {
|
||||||
|
rnnCoreAnnotationsPredictedF = SMX.getRnnCoreAnnotationsPredictedF()
|
||||||
|
}
|
||||||
|
if (rnnCoreAnnotationsPredicted1 == null) {
|
||||||
|
rnnCoreAnnotationsPredictedHashMap.put(str1, SMX.getRnnCoreAnnotationsPredicted1())
|
||||||
|
}
|
||||||
|
if (simpleMatricesF == null) {
|
||||||
|
simpleMatricesF = SMX.getSimpleMatricesF();
|
||||||
|
}
|
||||||
|
if (simpleMatrices1 == null) {
|
||||||
|
simpleMatricesHashMap.put(str1, SMX.getSimpleMatrices1());
|
||||||
|
}
|
||||||
|
if (simpleMatricesNodevectorsF == null) {
|
||||||
|
simpleMatricesNodevectorsF = SMX.getSimpleMatricesNodevectorsF();
|
||||||
|
}
|
||||||
|
if (simpleMatricesNodevectors1 == null) {
|
||||||
|
simpleMatricesNodevectorsHashMap.put(str1, SMX.getSimpleMatricesNodevectors1());
|
||||||
|
}
|
||||||
|
if (listF == null) {
|
||||||
|
listF = SMX.getListF();
|
||||||
|
}
|
||||||
|
if (list1 == null) {
|
||||||
|
listHashMap.put(str1, SMX.getList1());
|
||||||
|
}
|
||||||
|
if (longestF == null) {
|
||||||
|
longestF = SMX.getLongestF();
|
||||||
|
}
|
||||||
|
if (longest1 == null) {
|
||||||
|
longestHashMap.put(str1, SMX.getLongest1());
|
||||||
|
}
|
||||||
|
if (sentimentLongestF == null) {
|
||||||
|
sentimentLongestF = SMX.getSentimentLongestF();
|
||||||
|
}
|
||||||
|
if (sentimentLongest1 == null) {
|
||||||
|
sentimentHashMap.put(str1, SMX.getSentimentLongest1());
|
||||||
|
}
|
||||||
|
if (imwesF == null) {
|
||||||
|
imwesF = SMX.getImwesF();
|
||||||
|
}
|
||||||
|
if (imwes1 == null) {
|
||||||
|
imwesHashMap.put(str1, SMX.getImwes1());
|
||||||
|
}
|
||||||
|
if (InflectedCounterNegativeF == null) {
|
||||||
|
InflectedCounterNegativeF = SMX.getInflectedCounterNegativeF();
|
||||||
|
}
|
||||||
|
if (InflectedCounterNegative1 == null) {
|
||||||
|
InflectedCounterNegativeHashMap.put(str1, SMX.getInflectedCounterNegative1());
|
||||||
|
}
|
||||||
|
if (InflectedCounterPositiveF == null) {
|
||||||
|
InflectedCounterPositiveF = SMX.getInflectedCounterPositiveF();
|
||||||
|
}
|
||||||
|
if (InflectedCounterPositive1 == null) {
|
||||||
|
InflectedCounterPositiveHashMap.put(str1, SMX.getInflectedCounterPositive1());
|
||||||
|
}
|
||||||
|
if (tokenEntryF == null) {
|
||||||
|
tokenEntryF = SMX.getTokenEntryF();
|
||||||
|
}
|
||||||
|
if (tokenEntry1 == null) {
|
||||||
|
tokenEntryHashMap.put(str1, SMX.getTokenEntry1())
|
||||||
|
}
|
||||||
|
if (MarkedContinuousCounterF == null) {
|
||||||
|
MarkedContinuousCounterF = SMX.getMarkedContinuousCounterF();
|
||||||
|
}
|
||||||
|
if (MarkedContinuousCounter1 == null) {
|
||||||
|
MarkedContinuousCounterHashMap.put(str1, SMX.getMarkedContinuousCounter1());
|
||||||
|
}
|
||||||
|
if (UnmarkedPatternCounterF == null) {
|
||||||
|
UnmarkedPatternCounterF = SMX.getUnmarkedPatternCounterF();
|
||||||
|
}
|
||||||
|
if (UnmarkedPatternCounter1 == null) {
|
||||||
|
UnmarkedPatternCounterHashMap.put(str1, SMX.getUnmarkedPatternCounter1());
|
||||||
|
}
|
||||||
|
if (strTokensIpartFormF == null) {
|
||||||
|
strTokensIpartFormF = SMX.getStrTokensIpartFormF();
|
||||||
|
}
|
||||||
|
if (strTokensIpartForm1 == null) {
|
||||||
|
strTokensIpartFormHashMap.put(str1, SMX.getStrTokensIpartForm1());
|
||||||
|
}
|
||||||
|
if (tokenFormsF == null) {
|
||||||
|
tokenFormsF = SMX.getTokenFormsF();
|
||||||
|
}
|
||||||
|
if (tokenForms1 == null) {
|
||||||
|
tokenFormsHashMap.put(str1, SMX.getTokenForms1());
|
||||||
|
}
|
||||||
|
if (strTokenEntryGetPOSF == null) {
|
||||||
|
strTokenEntryGetPOSF = SMX.getStrTokenEntryGetPOSF();
|
||||||
|
}
|
||||||
|
if (strTokenEntryGetPOS1 == null) {
|
||||||
|
strTokenEntryGetPOSHashMap.put(str1, SMX.getStrTokenEntryGetPOS1())
|
||||||
|
}
|
||||||
|
if (intTokenEntyCountsF == null) {
|
||||||
|
intTokenEntyCountsF = SMX.getIntTokenEntyCountsF();
|
||||||
|
}
|
||||||
|
if (intTokenEntyCounts1 == null) {
|
||||||
|
intTokenEntyCountsHashMap.put(str1, SMX.getIntTokenEntyCounts1());
|
||||||
|
}
|
||||||
|
if (ITokenTagsF == null) {
|
||||||
|
ITokenTagsF = SMX.getITokenTagsF();
|
||||||
|
}
|
||||||
|
if (ITokenTags1 == null) {
|
||||||
|
ITokenTagsHashMap.put(str1, SMX.getITokenTags1());
|
||||||
|
}
|
||||||
|
if (strTokenStemsF == null) {
|
||||||
|
strTokenStemsF = SMX.getStrTokenStemsF();
|
||||||
|
}
|
||||||
|
if (strTokenStems1 == null) {
|
||||||
|
strTokenStemsHashMap.put(str1, SMX.getStrTokenStems1());
|
||||||
|
}
|
||||||
|
if (AnotatorcounterF == null) {
|
||||||
|
AnotatorcounterF = SMX.getAnotatorcounterF();
|
||||||
|
}
|
||||||
|
if (Anotatorcounter1 == null) {
|
||||||
|
AnotatorcounterHashMap.put(str1, SMX.getAnotatorcounter1());
|
||||||
|
}
|
||||||
|
if (TokensCounterF == null) {
|
||||||
|
TokensCounterF = SMX.getTokensCounterF();
|
||||||
|
}
|
||||||
|
if (TokensCounter1 == null) {
|
||||||
|
TokensCounterHashMap.put(str1, SMX.getTokensCounter1());
|
||||||
|
}
|
||||||
|
if (entityTokenTagsF == null) {
|
||||||
|
entityTokenTagsF = SMX.getEntityTokenTagsF();
|
||||||
|
}
|
||||||
|
if (entityTokenTags1 == null) {
|
||||||
|
entityTokenTagsHashMap.put(str1, SMX.getEntityTokenTags1());
|
||||||
|
}
|
||||||
|
if (nerEntitiesF == null) {
|
||||||
|
nerEntitiesF = SMX.getNerEntitiesF();
|
||||||
|
}
|
||||||
|
if (nerEntities1 == null) {
|
||||||
|
nerEntitiesHashMap.put(str1, SMX.getNerEntities1());
|
||||||
|
}
|
||||||
|
if (nerEntitiesTypeF == null) {
|
||||||
|
nerEntitiesTypeF = SMX.getNerEntitiesTypeF();
|
||||||
|
}
|
||||||
|
if (nerEntitiesType1 == null) {
|
||||||
|
nerEntitiesTypeHashMap.put(str1, SMX.getNerEntitiesType1());
|
||||||
|
}
|
||||||
|
if (stopWordTokenF == null) {
|
||||||
|
stopWordTokenF = SMX.getStopWordTokenF();
|
||||||
|
}
|
||||||
|
if (stopWordToken1 == null) {
|
||||||
|
stopWordTokenHashMap.put(str1, SMX.getStopWordToken1());
|
||||||
|
}
|
||||||
|
if (stopWordLemmaF == null) {
|
||||||
|
stopWordLemmaF = SMX.getStopWordLemmaF();
|
||||||
|
}
|
||||||
|
if (stopWordLemma1 == null) {
|
||||||
|
stopWordLemmaHashMap.put(str1, SMX.getStopWordLemma1());
|
||||||
|
}
|
||||||
|
if (PairCounterF == null) {
|
||||||
|
PairCounterF = SMX.getPairCounterF();
|
||||||
|
}
|
||||||
|
if (PairCounter1 == null) {
|
||||||
|
PairCounterHashMap.put(str1, SMX.getPairCounter1());
|
||||||
|
}
|
||||||
|
|
||||||
|
var getSMX: SimilarityMatrix = SMX.callSMX()
|
||||||
|
val scoreRelationLastUserMsg = getSMX.distance
|
||||||
|
if (scoreRelationLastUserMsg > preRelationUserCounters) {
|
||||||
|
preRelationUserCounters = scoreRelationLastUserMsg
|
||||||
|
concurrentRelations.add(getSMX.secondaryString)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val cacheRequirement = 6500;
|
||||||
|
if (preRelationUserCounters > cacheRequirement && !values_copy.contains(strF) && filterContent(strF)) {
|
||||||
|
DataMapper.InsertMYSQLStrings(strF)
|
||||||
|
DataMapper.checkStringsToDelete();
|
||||||
|
}
|
||||||
|
val randomLenghtPermit = strF.length * (Math.random() * Math.random() * Math.random() * (Math.random() * 10))
|
||||||
|
Collections.reverse(concurrentRelations)
|
||||||
|
val mysqlUpdateLastUsed: ArrayList<String> = ArrayList()
|
||||||
|
if (!concurrentRelations.isEmpty()) {
|
||||||
|
for (secondaryRelation in concurrentRelations) {
|
||||||
|
if (SB.toString().length > randomLenghtPermit && !SB.toString().isEmpty()) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
SB.append(secondaryRelation).append(" ")
|
||||||
|
mysqlUpdateLastUsed.add(secondaryRelation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (SB.toString().isEmpty()) {
|
||||||
|
return "failure, preventing stuckness"
|
||||||
|
}
|
||||||
|
DataMapper.updateLastUsed(mysqlUpdateLastUsed);
|
||||||
|
return SB.toString()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun getJMWEAnnotation(str1: String) {
|
||||||
|
val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str1)
|
||||||
|
jmweAnnotationCache.put(str1, jmweAnnotation)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getResponseMsg(str: String, personName: String, stanfordCoreNLP: StanfordCoreNLP,
|
||||||
|
stanfordCoreNLPSentiment: StanfordCoreNLP, ingameResponse: Boolean): String {
|
||||||
|
var responseFutures: String = ""
|
||||||
|
runBlocking {
|
||||||
|
launch(Dispatchers.Default) {
|
||||||
|
var strF = trimString(str)
|
||||||
|
responseFutures = getResponseFutures(strF, stanfordCoreNLP, stanfordCoreNLPSentiment)
|
||||||
|
if (!ingameResponse) {
|
||||||
|
responseFutures = checkPersonPresentInSentence(personName, responseFutures, strF, stanfordCoreNLP,
|
||||||
|
stanfordCoreNLPSentiment)
|
||||||
|
}
|
||||||
|
yield()
|
||||||
|
}.join()
|
||||||
|
}
|
||||||
|
return responseFutures
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun checkPersonPresentInSentence(personName: String, responseMsg: String, userLastMessage: String,
|
||||||
|
stanfordCoreNLP: StanfordCoreNLP,
|
||||||
|
stanfordCoreNLPSentiment: StanfordCoreNLP): String {
|
||||||
|
try {
|
||||||
|
val pipelineCoreDcoument = CoreDocument(responseMsg)
|
||||||
|
val pipelineCoreDcoumentLastMsg = CoreDocument(userLastMessage)
|
||||||
|
stanfordCoreNLP.annotate(pipelineCoreDcoument)
|
||||||
|
stanfordCoreNLPSentiment.annotate(pipelineCoreDcoumentLastMsg)
|
||||||
|
val regex = "(.*?\\d){10,}"
|
||||||
|
for (em in pipelineCoreDcoument.entityMentions()) {
|
||||||
|
val entityType = em.entityType()
|
||||||
|
if (entityType == "PERSON") {
|
||||||
|
var str = responseMsg
|
||||||
|
val emText = em.text()
|
||||||
|
val pattern = Pattern.compile(regex)
|
||||||
|
val matcher = pattern.matcher(personName)
|
||||||
|
val isMatched = matcher.matches()
|
||||||
|
if (emText != personName && !isMatched) {
|
||||||
|
for (emLastMsg in pipelineCoreDcoumentLastMsg.entityMentions()) {
|
||||||
|
if (emText != emLastMsg.text() && !Character.isDigit(emLastMsg.text().trim { it <= ' ' }[0])) {
|
||||||
|
str = (responseMsg.substring(0, responseMsg.indexOf(emText)) + " "
|
||||||
|
+ emLastMsg + " " + responseMsg.substring(responseMsg.indexOf(emText)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
str += " $personName"
|
||||||
|
return str
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e: Exception) {
|
||||||
|
println("""SCUFFED JAYZ: ${e.message}""".trimIndent())
|
||||||
|
}
|
||||||
|
return responseMsg
|
||||||
|
}
|
||||||
|
|
||||||
|
fun filterContent(str: String): Boolean {
|
||||||
|
if (!str.isEmpty() && str.length > 3) {
|
||||||
|
var str1Local: String = str.trim();
|
||||||
|
if (str1Local.length > 2 && !str1Local.startsWith("!")) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getCoreDocumentsSuggested(pipeline: StanfordCoreNLP, str: String) {
|
||||||
|
val annotation = Annotation(str)
|
||||||
|
pipeline.annotate(annotation)
|
||||||
|
val coreDocument = CoreDocument(annotation)
|
||||||
|
coreDocumentAnnotationCache.put(str, coreDocument)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -5,20 +5,13 @@
|
|||||||
*/
|
*/
|
||||||
package FunctionLayer;
|
package FunctionLayer;
|
||||||
|
|
||||||
import PresentationLayer.DiscordHandler;
|
|
||||||
import discord4j.core.event.domain.message.MessageCreateEvent;
|
import discord4j.core.event.domain.message.MessageCreateEvent;
|
||||||
import discord4j.core.object.entity.User;
|
import discord4j.core.object.entity.User;
|
||||||
import discord4j.core.object.entity.channel.TextChannel;
|
import discord4j.core.object.entity.channel.TextChannel;
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.logging.Level;
|
|
||||||
import java.util.logging.Logger;
|
|
||||||
|
|
||||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
||||||
import reactor.core.publisher.Flux;
|
|
||||||
import reactor.core.publisher.Mono;
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
import FunctionLayer.Datahandler;
|
import FunctionLayer.DatahandlerKotlinObsolete;
|
||||||
import FunctionLayer.PipelineJMWESingleton;
|
import FunctionLayer.PipelineJMWESingleton;
|
||||||
import FunctionLayer.StanfordParser.SentimentAnalyzerTest;
|
import FunctionLayer.StanfordParser.SentimentAnalyzerTest;
|
||||||
import edu.mit.jmwe.data.IMWE;
|
import edu.mit.jmwe.data.IMWE;
|
||||||
@ -17,10 +17,9 @@ import edu.stanford.nlp.trees.*;
|
|||||||
import edu.stanford.nlp.util.CoreMap;
|
import edu.stanford.nlp.util.CoreMap;
|
||||||
import org.ejml.simple.SimpleMatrix;
|
import org.ejml.simple.SimpleMatrix;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
|
||||||
import FunctionLayer.SimilarityMatrix;
|
import FunctionLayer.SimilarityMatrix;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@ -178,7 +177,7 @@ public class junit {
|
|||||||
|
|
||||||
//@Test
|
//@Test
|
||||||
public void testScoring() {
|
public void testScoring() {
|
||||||
Datahandler datahandler = new Datahandler();
|
DatahandlerKotlinObsolete datahandler = new DatahandlerKotlinObsolete();
|
||||||
PipelineJMWESingleton.getINSTANCE();
|
PipelineJMWESingleton.getINSTANCE();
|
||||||
StanfordCoreNLP stanfordCoreNLP = datahandler.pipeLineSetUp();
|
StanfordCoreNLP stanfordCoreNLP = datahandler.pipeLineSetUp();
|
||||||
StanfordCoreNLP stanfordCoreNLPSentiment = datahandler.shiftReduceParserInitiate();
|
StanfordCoreNLP stanfordCoreNLPSentiment = datahandler.shiftReduceParserInitiate();
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user