redid kotlin to java and removed coroutines again
This commit is contained in:
christian
parent
8eacf7da71
commit
d7ed45922e
@ -20,7 +20,7 @@ import java.util.logging.Logger;
|
||||
*/
|
||||
public class DataMapper {
|
||||
|
||||
public static ArrayList<String> getAllStrings() throws SQLException {
|
||||
public static ArrayList<String> getAllStrings() {
|
||||
Connection l_cCon = null;
|
||||
PreparedStatement l_pStatement = null;
|
||||
ResultSet l_rsSearch = null;
|
||||
@ -33,13 +33,15 @@ public class DataMapper {
|
||||
while (l_rsSearch.next()) {
|
||||
arrayListStr.add(l_rsSearch.getString(1));
|
||||
}
|
||||
} catch (SQLException throwables) {
|
||||
throwables.printStackTrace();
|
||||
} finally {
|
||||
CloseConnections(l_pStatement, l_rsSearch, l_cCon);
|
||||
}
|
||||
return arrayListStr;
|
||||
}
|
||||
|
||||
public static void InsertMYSQLStrings(String str) throws SQLException {
|
||||
public static void InsertMYSQLStrings(String str) {
|
||||
Connection l_cCon = null;
|
||||
PreparedStatement l_pStatement = null;
|
||||
ResultSet l_rsSearch = null;
|
||||
@ -49,6 +51,8 @@ public class DataMapper {
|
||||
l_pStatement = l_cCon.prepareStatement(l_sSQL);
|
||||
l_pStatement.setString(1, str);
|
||||
l_pStatement.execute();
|
||||
} catch (SQLException throwables) {
|
||||
throwables.printStackTrace();
|
||||
} finally {
|
||||
CloseConnections(l_pStatement, l_rsSearch, l_cCon);
|
||||
}
|
||||
|
||||
677
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
Normal file
677
ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java
Normal file
@ -0,0 +1,677 @@
|
||||
package FunctionLayer;
|
||||
|
||||
import DataLayer.DataMapper;
|
||||
import FunctionLayer.StanfordParser.SentimentAnalyzerTest;
|
||||
import edu.mit.jmwe.data.IMWE;
|
||||
import edu.mit.jmwe.data.IToken;
|
||||
import edu.stanford.nlp.ie.AbstractSequenceClassifier;
|
||||
import edu.stanford.nlp.ie.crf.CRFClassifier;
|
||||
import edu.stanford.nlp.ie.machinereading.structure.EntityMention;
|
||||
import edu.stanford.nlp.ling.CoreAnnotations;
|
||||
import edu.stanford.nlp.ling.CoreLabel;
|
||||
import edu.stanford.nlp.ling.TaggedWord;
|
||||
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
|
||||
import edu.stanford.nlp.pipeline.Annotation;
|
||||
import edu.stanford.nlp.pipeline.CoreDocument;
|
||||
import edu.stanford.nlp.pipeline.CoreEntityMention;
|
||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
||||
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
|
||||
import edu.stanford.nlp.trees.*;
|
||||
import edu.stanford.nlp.util.CoreMap;
|
||||
import kotlinx.coroutines.*;
|
||||
import org.ejml.simple.SimpleMatrix;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
public class Datahandler {
|
||||
private HashMap<String, Annotation> pipelineAnnotationCache;
|
||||
private HashMap<String, Annotation> pipelineSentimentAnnotationCache;
|
||||
private HashMap<String, CoreDocument> coreDocumentAnnotationCache;
|
||||
private HashMap<String, Annotation> jmweAnnotationCache;
|
||||
|
||||
private MaxentTagger tagger = new MaxentTagger();
|
||||
|
||||
private GrammaticalStructureFactory gsf;
|
||||
private AbstractSequenceClassifier<CoreLabel> classifier;
|
||||
|
||||
//SentimentAnalyzer Hashmaps
|
||||
private HashMap<String, Integer> tokenizeCountingHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, List<List<TaggedWord>>> taggedWordListHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> retrieveTGWListHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, List<CoreMap>> sentences1HashMap = new HashMap();
|
||||
|
||||
private HashMap<String, List<CoreMap>> sentencesSentimentHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<Tree>> trees1HashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<GrammaticalStructure>> grammaticalStructureHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<TypedDependency>> typedDependenciesHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<Integer>> rnnCoreAnnotationsPredictedHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<SimpleMatrix>> simpleMatricesHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<SimpleMatrix>> simpleMatricesNodevectorsHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, List> listHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, Integer> longestHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, Integer> sentimentHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, List<IMWE<IToken>>> imwesHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, Integer> InflectedCounterNegativeHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, Integer> InflectedCounterPositiveHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, ArrayList<String>> tokenEntryHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, Integer> MarkedContinuousCounterHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, Integer> UnmarkedPatternCounterHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> strTokensIpartFormHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> tokenFormsHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> strTokenEntryGetPOSHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<Integer>> intTokenEntyCountsHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> ITokenTagsHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> strTokenStemsHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, Integer> AnotatorcounterHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, Integer> TokensCounterHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> entityTokenTagsHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> nerEntitiesHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> nerEntitiesTypeHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> stopWordTokenHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, java.util.ArrayList<String>> stopWordLemmaHashMap = new HashMap();
|
||||
|
||||
private HashMap<String, Integer> PairCounterHashMap = new HashMap();
|
||||
|
||||
public Datahandler() {
|
||||
jmweAnnotationCache = new HashMap<String, Annotation>();
|
||||
pipelineAnnotationCache = new HashMap<String, Annotation>();
|
||||
pipelineSentimentAnnotationCache = new HashMap<String, Annotation>();
|
||||
coreDocumentAnnotationCache = new HashMap<String, CoreDocument>();
|
||||
gsf = initiateGrammaticalStructureFactory();
|
||||
String nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz";
|
||||
classifier = CRFClassifier.getClassifierNoExceptions(nerModel);
|
||||
}
|
||||
|
||||
private GrammaticalStructureFactory initiateGrammaticalStructureFactory() {
|
||||
// lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
|
||||
String lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";
|
||||
LexicalizedParser lp = LexicalizedParser.loadModel(lexParserEnglishPCFG, "-maxLength", "100");
|
||||
TreebankLanguagePack tlp = lp.getOp().langpack();
|
||||
return tlp.grammaticalStructureFactory();
|
||||
}
|
||||
|
||||
public StanfordCoreNLP pipeLineSetUp() {
|
||||
Properties props = new Properties();
|
||||
String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz";
|
||||
// nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz"
|
||||
//String nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz";
|
||||
// nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz"
|
||||
//String nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.distsim.crf.ser.gz";
|
||||
props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse");
|
||||
props.setProperty("parse.model", shiftReduceParserPath);
|
||||
props.setProperty("parse.maxlen", "90");
|
||||
props.setProperty("parse.binaryTrees", "true");
|
||||
props.setProperty("threads", "5");
|
||||
props.setProperty("pos.maxlen", "90");
|
||||
props.setProperty("tokenize.maxlen", "90");
|
||||
props.setProperty("ssplit.maxlen", "90");
|
||||
props.setProperty("lemma.maxlen", "90");
|
||||
props.setProperty("ner.model", "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz" +
|
||||
",edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz" +
|
||||
",edu/stanford/nlp/models/ner/english.muc.7class.distsim.crf.ser.gz");
|
||||
props.setProperty("ner.combinationMode", "HIGH_RECALL");
|
||||
props.setProperty("regexner.ignorecase", "true");
|
||||
props.setProperty("ner.fine.regexner.ignorecase", "true");
|
||||
props.setProperty("tokenize.options", "untokenizable=firstKeep");
|
||||
return new StanfordCoreNLP(props);
|
||||
}
|
||||
|
||||
public StanfordCoreNLP shiftReduceParserInitiate() {
|
||||
Properties propsSentiment = new Properties();
|
||||
// lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
|
||||
String lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";
|
||||
String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz";
|
||||
// taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger"
|
||||
String taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words-distsim.tagger";
|
||||
String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of," +
|
||||
"on,or,such,that,the,their,then,there,these,they,this,to,was,will,with";
|
||||
propsSentiment.setProperty("parse.model", lexParserEnglishPCFG);
|
||||
propsSentiment.setProperty("sentiment.model", sentimentModel);
|
||||
propsSentiment.setProperty("parse.maxlen", "90");
|
||||
propsSentiment.setProperty("threads", "5");
|
||||
propsSentiment.setProperty("pos.maxlen", "90");
|
||||
propsSentiment.setProperty("tokenize.maxlen", "90");
|
||||
propsSentiment.setProperty("ssplit.maxlen", "90");
|
||||
propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword"); //coref too expensive memorywise
|
||||
propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator");
|
||||
propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList);
|
||||
propsSentiment.setProperty("tokenize.options", "untokenizable=firstKeep");
|
||||
tagger = new MaxentTagger(taggerPath);
|
||||
return new StanfordCoreNLP(propsSentiment);
|
||||
}
|
||||
|
||||
private String trimString(String str) {
|
||||
String message = str.trim();
|
||||
if (message.startsWith("<@")) {
|
||||
message = message.substring(message.indexOf("> ") + 2);
|
||||
}
|
||||
if (!message.isEmpty()) {
|
||||
message = message.replace("@", "");
|
||||
if (message.contains("<>")) {
|
||||
message = message.substring(message.indexOf(">"));
|
||||
}
|
||||
if (message.startsWith("[ *")) {
|
||||
message = message.substring(message.indexOf("]"));
|
||||
}
|
||||
}
|
||||
return message;
|
||||
}
|
||||
|
||||
private void createStrAnnotation(String str, StanfordCoreNLP stanfordCoreNLP, Boolean sentimentBool) {
|
||||
Annotation strAnno2 = new Annotation(str);
|
||||
strAnno2.compact();
|
||||
stanfordCoreNLP.annotate(strAnno2);
|
||||
if (sentimentBool) {
|
||||
pipelineSentimentAnnotationCache.put(str, strAnno2);
|
||||
} else {
|
||||
pipelineAnnotationCache.put(str, strAnno2);
|
||||
}
|
||||
}
|
||||
|
||||
private String getResponseFutures(String strF, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) {
|
||||
|
||||
Annotation strAnno = new Annotation(strF);
|
||||
strAnno.compact();
|
||||
stanfordCoreNLP.annotate(strAnno);
|
||||
|
||||
Annotation strAnnoSentiment = new Annotation(strF);
|
||||
strAnnoSentiment.compact();
|
||||
stanfordCoreNLPSentiment.annotate(strAnnoSentiment);
|
||||
|
||||
Annotation annotation = new Annotation(strF);
|
||||
stanfordCoreNLP.annotate(annotation);
|
||||
CoreDocument coreDocument = new CoreDocument(annotation);
|
||||
|
||||
List<String> ues_copy = new ArrayList(DataMapper.getAllStrings());
|
||||
double preRelationUserCounters = -155000.0;
|
||||
ArrayList<String> concurrentRelations = new ArrayList();
|
||||
StringBuilder SB = new StringBuilder();
|
||||
Annotation jmweAnnotationF = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strF);
|
||||
Integer tokenizeCountingF = null;
|
||||
List<List<TaggedWord>> taggedWordListF = null;
|
||||
java.util.ArrayList<String> retrieveTGWListF = null;
|
||||
List<CoreMap> sentencesF = null;
|
||||
List<CoreMap> sentencesSentimentF = null;
|
||||
|
||||
List<CoreMap> coreMaps1 = jmweAnnotationF.get(CoreAnnotations.SentencesAnnotation.class);
|
||||
|
||||
java.util.ArrayList<Tree> treesF = null;
|
||||
|
||||
ArrayList<GrammaticalStructure> grammaticalStructuresF = null;
|
||||
|
||||
java.util.ArrayList<TypedDependency> typedDependenciesF = null;
|
||||
|
||||
java.util.ArrayList<Integer> rnnCoreAnnotationsPredictedF = null;
|
||||
|
||||
java.util.ArrayList<SimpleMatrix> simpleMatricesF = null;
|
||||
|
||||
java.util.ArrayList<SimpleMatrix> simpleMatricesNodevectorsF = null;
|
||||
List<String> listF = null;
|
||||
|
||||
Integer longestF = null;
|
||||
|
||||
Integer sentimentLongestF = null;
|
||||
|
||||
List<IMWE<IToken>> imwesF = null;
|
||||
|
||||
Integer InflectedCounterNegativeF = null;
|
||||
|
||||
Integer InflectedCounterPositiveF = null;
|
||||
|
||||
ArrayList<String> tokenEntryF = null;
|
||||
|
||||
Integer MarkedContinuousCounterF = null;
|
||||
|
||||
Integer UnmarkedPatternCounterF = null;
|
||||
|
||||
ArrayList<String> strTokensIpartFormF = null;
|
||||
|
||||
java.util.ArrayList<String> tokenFormsF = null;
|
||||
|
||||
ArrayList<String> strTokenEntryGetPOSF = null;
|
||||
|
||||
java.util.ArrayList<Integer> intTokenEntyCountsF = null;
|
||||
|
||||
ArrayList<String> ITokenTagsF = null;
|
||||
|
||||
java.util.ArrayList<String> strTokenStemsF = null;
|
||||
|
||||
Integer AnotatorcounterF = null;
|
||||
|
||||
Integer TokensCounterF = null;
|
||||
|
||||
java.util.ArrayList<String> entityTokenTagsF = null;
|
||||
|
||||
java.util.ArrayList<String> nerEntitiesF = null;
|
||||
|
||||
java.util.ArrayList<String> nerEntitiesTypeF = null;
|
||||
|
||||
java.util.ArrayList<String> stopWordTokenF = null;
|
||||
|
||||
java.util.ArrayList<String> stopWordLemmaF = null;
|
||||
|
||||
Integer PairCounterF = null;
|
||||
for (String str1 : ues_copy) {
|
||||
if (strF != str1) {
|
||||
Annotation annotation2 = pipelineSentimentAnnotationCache.getOrDefault(str1, null);
|
||||
Annotation annotation4 = pipelineAnnotationCache.getOrDefault(str1, null);
|
||||
CoreDocument coreDocument1 = coreDocumentAnnotationCache.getOrDefault(str1, null);
|
||||
Annotation jmweAnnotation = jmweAnnotationCache.getOrDefault(str1, null);
|
||||
if (annotation2 == null) {
|
||||
createStrAnnotation(str1, stanfordCoreNLPSentiment, true);
|
||||
}
|
||||
if (annotation4 == null) {
|
||||
createStrAnnotation(str1, stanfordCoreNLP, false);
|
||||
}
|
||||
if (coreDocument1 == null) {
|
||||
getCoreDocumentsSuggested(stanfordCoreNLP, str1);
|
||||
}
|
||||
if (jmweAnnotation == null) {
|
||||
getJMWEAnnotation(str1);
|
||||
jmweAnnotation = jmweAnnotationCache.get(str1);
|
||||
}
|
||||
Integer tokenizeCounting = tokenizeCountingHashMap.getOrDefault(str1, null);
|
||||
|
||||
List<List<TaggedWord>> taggedWordList1 = taggedWordListHashMap.getOrDefault(str1, null);
|
||||
|
||||
java.util.ArrayList<String> retrieveTGWList1 = retrieveTGWListHashMap.getOrDefault(str1, null);
|
||||
|
||||
List<CoreMap> sentence1 = sentences1HashMap.getOrDefault(str1, null);
|
||||
|
||||
List<CoreMap> sentenceSentiment1 = sentencesSentimentHashMap.getOrDefault(str1, null);
|
||||
ArrayList<Tree> trees1 = trees1HashMap.getOrDefault(str1, null);
|
||||
List<CoreMap> coreMaps2 = new ArrayList<>();
|
||||
ArrayList<GrammaticalStructure> grammaticalStructures1 = grammaticalStructureHashMap.getOrDefault(str1, null);
|
||||
if (jmweAnnotation != null) {
|
||||
coreMaps2 = jmweAnnotation.get(CoreAnnotations.SentencesAnnotation.class);
|
||||
}
|
||||
ArrayList<TypedDependency> typedDependencies1 = typedDependenciesHashMap.getOrDefault(str1, null);
|
||||
ArrayList<Integer> rnnCoreAnnotationsPredicted1 = rnnCoreAnnotationsPredictedHashMap.getOrDefault(str1, null);
|
||||
ArrayList<SimpleMatrix> simpleMatrices1 = simpleMatricesHashMap.getOrDefault(str1, null);
|
||||
simpleMatricesHashMap.getOrDefault(str1, null);
|
||||
ArrayList<SimpleMatrix> simpleMatricesNodevectors1 = simpleMatricesNodevectorsHashMap.getOrDefault(str1, null);
|
||||
List list1 = listHashMap.getOrDefault(str1, null);
|
||||
Integer longest1 = longestHashMap.getOrDefault(str1, null);
|
||||
Integer sentimentLongest1 = sentimentHashMap.getOrDefault(str1, null);
|
||||
List<IMWE<IToken>> imwes1 = imwesHashMap.getOrDefault(str1, null);
|
||||
Integer InflectedCounterNegative1 = InflectedCounterNegativeHashMap.getOrDefault(str1, null);
|
||||
Integer InflectedCounterPositive1 = InflectedCounterPositiveHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> tokenEntry1 = tokenEntryHashMap.getOrDefault(str1, null);
|
||||
Integer MarkedContinuousCounter1 = MarkedContinuousCounterHashMap.getOrDefault(str1, null);
|
||||
Integer UnmarkedPatternCounter1 = UnmarkedPatternCounterHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> strTokensIpartForm1 = strTokensIpartFormHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> tokenForms1 = tokenFormsHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> strTokenEntryGetPOS1 = strTokenEntryGetPOSHashMap.getOrDefault(str1, null);
|
||||
ArrayList<Integer> intTokenEntyCounts1 = intTokenEntyCountsHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> ITokenTags1 = ITokenTagsHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> strTokenStems1 = strTokenStemsHashMap.getOrDefault(str1, null);
|
||||
Integer Anotatorcounter1 = AnotatorcounterHashMap.getOrDefault(str1, null);
|
||||
Integer TokensCounter1 = TokensCounterHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> entityTokenTags1 = entityTokenTagsHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> nerEntities1 = nerEntitiesHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> nerEntitiesType1 = nerEntitiesTypeHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> stopWordToken1 = stopWordTokenHashMap.getOrDefault(str1, null);
|
||||
ArrayList<String> stopWordLemma1 = stopWordLemmaHashMap.getOrDefault(str1, null);
|
||||
Integer PairCounter1 = PairCounterHashMap.getOrDefault(str1, null);
|
||||
|
||||
SentimentAnalyzerTest SMX = new SentimentAnalyzerTest(strF, str1, new SimilarityMatrix(strF, str1),
|
||||
coreMaps1, coreMaps2, strAnno,
|
||||
pipelineAnnotationCache.get(str1), strAnnoSentiment,
|
||||
pipelineSentimentAnnotationCache.get(str1), coreDocument, coreDocumentAnnotationCache.get(str1),
|
||||
tagger, gsf, classifier, tokenizeCounting, tokenizeCountingF,
|
||||
taggedWordListF, taggedWordList1, retrieveTGWListF, retrieveTGWList1,
|
||||
sentencesF, sentence1, sentencesSentimentF, sentenceSentiment1, treesF, trees1,
|
||||
grammaticalStructuresF, grammaticalStructures1, typedDependenciesF,
|
||||
typedDependencies1, rnnCoreAnnotationsPredictedF, rnnCoreAnnotationsPredicted1,
|
||||
simpleMatricesF, simpleMatrices1, simpleMatricesNodevectorsF, simpleMatricesNodevectors1,
|
||||
listF, list1, longestF, longest1, sentimentLongestF, sentimentLongest1, imwesF,
|
||||
imwes1, InflectedCounterNegativeF, InflectedCounterNegative1, InflectedCounterPositiveF,
|
||||
InflectedCounterPositive1, tokenEntryF, tokenEntry1, MarkedContinuousCounterF,
|
||||
MarkedContinuousCounter1, UnmarkedPatternCounterF, UnmarkedPatternCounter1,
|
||||
strTokensIpartFormF, strTokensIpartForm1, tokenFormsF, tokenForms1,
|
||||
strTokenEntryGetPOSF, strTokenEntryGetPOS1, intTokenEntyCountsF,
|
||||
intTokenEntyCounts1, ITokenTagsF, ITokenTags1, strTokenStemsF, strTokenStems1,
|
||||
AnotatorcounterF, Anotatorcounter1, TokensCounterF, TokensCounter1,
|
||||
entityTokenTagsF, entityTokenTags1, nerEntitiesF, nerEntities1, nerEntitiesTypeF,
|
||||
nerEntitiesType1, stopWordTokenF, stopWordToken1, stopWordLemmaF, stopWordLemma1,
|
||||
PairCounterF, PairCounter1
|
||||
);
|
||||
if (tokenizeCounting == null) {
|
||||
tokenizeCountingHashMap.put(str1, SMX.getTokenizeCounting());
|
||||
}
|
||||
if (taggedWordList1 == null) {
|
||||
taggedWordListHashMap.put(str1, SMX.getTaggedWordList1());
|
||||
}
|
||||
if (tokenizeCountingF == null) {
|
||||
tokenizeCountingF = SMX.getTokenizeCountingF();
|
||||
}
|
||||
if (taggedWordListF == null) {
|
||||
taggedWordListF = SMX.getTaggedWordListF();
|
||||
}
|
||||
if (retrieveTGWListF == null) {
|
||||
retrieveTGWListF = SMX.getRetrieveTGWListF();
|
||||
}
|
||||
if (retrieveTGWList1 == null) {
|
||||
retrieveTGWListHashMap.put(str1, SMX.getRetrieveTGWList1());
|
||||
}
|
||||
if (sentencesF == null) {
|
||||
sentencesF = SMX.getSentencesF();
|
||||
}
|
||||
if (sentence1 == null) {
|
||||
sentences1HashMap.put(str1, SMX.getSentences1());
|
||||
}
|
||||
if (sentencesSentimentF == null) {
|
||||
sentencesSentimentF = SMX.getSentencesSentimentF();
|
||||
}
|
||||
if (sentenceSentiment1 == null) {
|
||||
sentencesSentimentHashMap.put(str1, SMX.getSentencesSentiment1());
|
||||
}
|
||||
if (treesF == null) {
|
||||
treesF = SMX.getTreesF();
|
||||
}
|
||||
if (trees1 == null) {
|
||||
trees1HashMap.put(str1, SMX.getTrees1());
|
||||
}
|
||||
if (grammaticalStructuresF == null) {
|
||||
grammaticalStructuresF = SMX.getGrammaticalStructuresF();
|
||||
}
|
||||
if (grammaticalStructures1 == null) {
|
||||
grammaticalStructureHashMap.put(str1, SMX.getGrammaticalStructures1());
|
||||
}
|
||||
if (typedDependenciesF == null) {
|
||||
typedDependenciesF = SMX.getTypedDependenciesF();
|
||||
}
|
||||
if (typedDependencies1 == null) {
|
||||
typedDependenciesHashMap.put(str1, SMX.getTypedDependencies1());
|
||||
}
|
||||
if (rnnCoreAnnotationsPredictedF == null) {
|
||||
rnnCoreAnnotationsPredictedF = SMX.getRnnCoreAnnotationsPredictedF();
|
||||
}
|
||||
if (rnnCoreAnnotationsPredicted1 == null) {
|
||||
rnnCoreAnnotationsPredictedHashMap.put(str1, SMX.getRnnCoreAnnotationsPredicted1());
|
||||
}
|
||||
if (simpleMatricesF == null) {
|
||||
simpleMatricesF = SMX.getSimpleMatricesF();
|
||||
}
|
||||
if (simpleMatrices1 == null) {
|
||||
simpleMatricesHashMap.put(str1, SMX.getSimpleMatrices1());
|
||||
}
|
||||
if (simpleMatricesNodevectorsF == null) {
|
||||
simpleMatricesNodevectorsF = SMX.getSimpleMatricesNodevectorsF();
|
||||
}
|
||||
if (simpleMatricesNodevectors1 == null) {
|
||||
simpleMatricesNodevectorsHashMap.put(str1, SMX.getSimpleMatricesNodevectors1());
|
||||
}
|
||||
if (listF == null) {
|
||||
listF = SMX.getListF();
|
||||
}
|
||||
if (list1 == null) {
|
||||
listHashMap.put(str1, SMX.getList1());
|
||||
}
|
||||
if (longestF == null) {
|
||||
longestF = SMX.getLongestF();
|
||||
}
|
||||
if (longest1 == null) {
|
||||
longestHashMap.put(str1, SMX.getLongest1());
|
||||
}
|
||||
if (sentimentLongestF == null) {
|
||||
sentimentLongestF = SMX.getSentimentLongestF();
|
||||
}
|
||||
if (sentimentLongest1 == null) {
|
||||
sentimentHashMap.put(str1, SMX.getSentimentLongest1());
|
||||
}
|
||||
if (imwesF == null) {
|
||||
imwesF = SMX.getImwesF();
|
||||
}
|
||||
if (imwes1 == null) {
|
||||
imwesHashMap.put(str1, SMX.getImwes1());
|
||||
}
|
||||
if (InflectedCounterNegativeF == null) {
|
||||
InflectedCounterNegativeF = SMX.getInflectedCounterNegativeF();
|
||||
}
|
||||
if (InflectedCounterNegative1 == null) {
|
||||
InflectedCounterNegativeHashMap.put(str1, SMX.getInflectedCounterNegative1());
|
||||
}
|
||||
if (InflectedCounterPositiveF == null) {
|
||||
InflectedCounterPositiveF = SMX.getInflectedCounterPositiveF();
|
||||
}
|
||||
if (InflectedCounterPositive1 == null) {
|
||||
InflectedCounterPositiveHashMap.put(str1, SMX.getInflectedCounterPositive1());
|
||||
}
|
||||
if (tokenEntryF == null) {
|
||||
tokenEntryF = SMX.getTokenEntryF();
|
||||
}
|
||||
if (tokenEntry1 == null) {
|
||||
tokenEntryHashMap.put(str1, SMX.getTokenEntry1());
|
||||
}
|
||||
if (MarkedContinuousCounterF == null) {
|
||||
MarkedContinuousCounterF = SMX.getMarkedContinuousCounterF();
|
||||
}
|
||||
if (MarkedContinuousCounter1 == null) {
|
||||
MarkedContinuousCounterHashMap.put(str1, SMX.getMarkedContinuousCounter1());
|
||||
}
|
||||
if (UnmarkedPatternCounterF == null) {
|
||||
UnmarkedPatternCounterF = SMX.getUnmarkedPatternCounterF();
|
||||
}
|
||||
if (UnmarkedPatternCounter1 == null) {
|
||||
UnmarkedPatternCounterHashMap.put(str1, SMX.getUnmarkedPatternCounter1());
|
||||
}
|
||||
if (strTokensIpartFormF == null) {
|
||||
strTokensIpartFormF = SMX.getStrTokensIpartFormF();
|
||||
}
|
||||
if (strTokensIpartForm1 == null) {
|
||||
strTokensIpartFormHashMap.put(str1, SMX.getStrTokensIpartForm1());
|
||||
}
|
||||
if (tokenFormsF == null) {
|
||||
tokenFormsF = SMX.getTokenFormsF();
|
||||
}
|
||||
if (tokenForms1 == null) {
|
||||
tokenFormsHashMap.put(str1, SMX.getTokenForms1());
|
||||
}
|
||||
if (strTokenEntryGetPOSF == null) {
|
||||
strTokenEntryGetPOSF = SMX.getStrTokenEntryGetPOSF();
|
||||
}
|
||||
if (strTokenEntryGetPOS1 == null) {
|
||||
strTokenEntryGetPOSHashMap.put(str1, SMX.getStrTokenEntryGetPOS1());
|
||||
}
|
||||
if (intTokenEntyCountsF == null) {
|
||||
intTokenEntyCountsF = SMX.getIntTokenEntyCountsF();
|
||||
}
|
||||
if (intTokenEntyCounts1 == null) {
|
||||
intTokenEntyCountsHashMap.put(str1, SMX.getIntTokenEntyCounts1());
|
||||
}
|
||||
if (ITokenTagsF == null) {
|
||||
ITokenTagsF = SMX.getITokenTagsF();
|
||||
}
|
||||
if (ITokenTags1 == null) {
|
||||
ITokenTagsHashMap.put(str1, SMX.getITokenTags1());
|
||||
}
|
||||
if (strTokenStemsF == null) {
|
||||
strTokenStemsF = SMX.getStrTokenStemsF();
|
||||
}
|
||||
if (strTokenStems1 == null) {
|
||||
strTokenStemsHashMap.put(str1, SMX.getStrTokenStems1());
|
||||
}
|
||||
if (AnotatorcounterF == null) {
|
||||
AnotatorcounterF = SMX.getAnotatorcounterF();
|
||||
}
|
||||
if (Anotatorcounter1 == null) {
|
||||
AnotatorcounterHashMap.put(str1, SMX.getAnotatorcounter1());
|
||||
}
|
||||
if (TokensCounterF == null) {
|
||||
TokensCounterF = SMX.getTokensCounterF();
|
||||
}
|
||||
if (TokensCounter1 == null) {
|
||||
TokensCounterHashMap.put(str1, SMX.getTokensCounter1());
|
||||
}
|
||||
if (entityTokenTagsF == null) {
|
||||
entityTokenTagsF = SMX.getEntityTokenTagsF();
|
||||
}
|
||||
if (entityTokenTags1 == null) {
|
||||
entityTokenTagsHashMap.put(str1, SMX.getEntityTokenTags1());
|
||||
}
|
||||
if (nerEntitiesF == null) {
|
||||
nerEntitiesF = SMX.getNerEntitiesF();
|
||||
}
|
||||
if (nerEntities1 == null) {
|
||||
nerEntitiesHashMap.put(str1, SMX.getNerEntities1());
|
||||
}
|
||||
if (nerEntitiesTypeF == null) {
|
||||
nerEntitiesTypeF = SMX.getNerEntitiesTypeF();
|
||||
}
|
||||
if (nerEntitiesType1 == null) {
|
||||
nerEntitiesTypeHashMap.put(str1, SMX.getNerEntitiesType1());
|
||||
}
|
||||
if (stopWordTokenF == null) {
|
||||
stopWordTokenF = SMX.getStopWordTokenF();
|
||||
}
|
||||
if (stopWordToken1 == null) {
|
||||
stopWordTokenHashMap.put(str1, SMX.getStopWordToken1());
|
||||
}
|
||||
if (stopWordLemmaF == null) {
|
||||
stopWordLemmaF = SMX.getStopWordLemmaF();
|
||||
}
|
||||
if (stopWordLemma1 == null) {
|
||||
stopWordLemmaHashMap.put(str1, SMX.getStopWordLemma1());
|
||||
}
|
||||
if (PairCounterF == null) {
|
||||
PairCounterF = SMX.getPairCounterF();
|
||||
}
|
||||
if (PairCounter1 == null) {
|
||||
PairCounterHashMap.put(str1, SMX.getPairCounter1());
|
||||
}
|
||||
|
||||
SimilarityMatrix getSMX = SMX.callSMX();
|
||||
double scoreRelationLastUserMsg = getSMX.getDistance();
|
||||
if (scoreRelationLastUserMsg > preRelationUserCounters) {
|
||||
preRelationUserCounters = scoreRelationLastUserMsg;
|
||||
concurrentRelations.add(getSMX.getSecondaryString());
|
||||
}
|
||||
}
|
||||
}
|
||||
int cacheRequirement = 6500;
|
||||
if (preRelationUserCounters > cacheRequirement && !ues_copy.contains(strF) && filterContent(strF)) {
|
||||
DataMapper.InsertMYSQLStrings(strF);
|
||||
DataMapper.checkStringsToDelete();
|
||||
}
|
||||
double randomLenghtPermit = strF.length() * (Math.random() * Math.random() * Math.random() * (Math.random() * 10));
|
||||
Collections.reverse(concurrentRelations);
|
||||
ArrayList<String> mysqlUpdateLastUsed = new ArrayList();
|
||||
if (!concurrentRelations.isEmpty()) {
|
||||
for (String secondaryRelation : concurrentRelations) {
|
||||
if (SB.toString().length() > randomLenghtPermit && !SB.toString().isEmpty()) {
|
||||
break;
|
||||
}
|
||||
SB.append(secondaryRelation).append(" ");
|
||||
mysqlUpdateLastUsed.add(secondaryRelation);
|
||||
}
|
||||
}
|
||||
if (SB.toString().isEmpty()) {
|
||||
return "failure, preventing stuckness";
|
||||
}
|
||||
DataMapper.updateLastUsed(mysqlUpdateLastUsed);
|
||||
return SB.toString();
|
||||
}
|
||||
|
||||
private void getJMWEAnnotation(String str1) {
|
||||
Annotation jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str1);
|
||||
jmweAnnotationCache.put(str1, jmweAnnotation);
|
||||
}
|
||||
|
||||
public String getResponseMsg(String str, String personName, StanfordCoreNLP stanfordCoreNLP,
|
||||
StanfordCoreNLP stanfordCoreNLPSentiment, Boolean ingameResponse) {
|
||||
String responseFutures = "";
|
||||
String strF = trimString(str);
|
||||
responseFutures = getResponseFutures(strF, stanfordCoreNLP, stanfordCoreNLPSentiment);
|
||||
if (!ingameResponse) {
|
||||
responseFutures = checkPersonPresentInSentence(personName, responseFutures, strF, stanfordCoreNLP,
|
||||
stanfordCoreNLPSentiment);
|
||||
}
|
||||
return responseFutures;
|
||||
}
|
||||
|
||||
private String checkPersonPresentInSentence(String personName, String responseMsg, String userLastMessage,
|
||||
StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) {
|
||||
try {
|
||||
CoreDocument pipelineCoreDcoument = new CoreDocument(responseMsg);
|
||||
CoreDocument pipelineCoreDcoumentLastMsg = new CoreDocument(userLastMessage);
|
||||
stanfordCoreNLP.annotate(pipelineCoreDcoument);
|
||||
stanfordCoreNLPSentiment.annotate(pipelineCoreDcoumentLastMsg);
|
||||
String regex = "(.*?\\d){10,}";
|
||||
for (CoreEntityMention em : pipelineCoreDcoument.entityMentions()) {
|
||||
String entityType = em.entityType();
|
||||
if (entityType == "PERSON") {
|
||||
String str = responseMsg;
|
||||
String emText = em.text();
|
||||
Pattern pattern = Pattern.compile(regex);
|
||||
Matcher matcher = pattern.matcher(personName);
|
||||
boolean isMatched = matcher.matches();
|
||||
if (emText != personName && !isMatched) {
|
||||
for (CoreEntityMention emLastMsg : pipelineCoreDcoumentLastMsg.entityMentions()) {
|
||||
if (emText != emLastMsg.text() && !Character.isDigit(Integer.parseInt(emLastMsg.text().trim()))) {
|
||||
str = (responseMsg.substring(0, responseMsg.indexOf(emText)) + " "
|
||||
+ emLastMsg + " " + responseMsg.substring(responseMsg.indexOf(emText)));
|
||||
}
|
||||
}
|
||||
str += personName;
|
||||
return str;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.out.println("SCUFFED JAYZ: " + e.getMessage());
|
||||
}
|
||||
return responseMsg;
|
||||
}
|
||||
|
||||
public boolean filterContent(String str) {
|
||||
if (!str.isEmpty() && str.length() > 3) {
|
||||
String str1Local = str.trim();
|
||||
if (str1Local.length() > 2 && !str1Local.startsWith("!")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void getCoreDocumentsSuggested(StanfordCoreNLP pipeline, String str) {
|
||||
Annotation annotation = new Annotation(str);
|
||||
pipeline.annotate(annotation);
|
||||
CoreDocument coreDocument = new CoreDocument(annotation);
|
||||
coreDocumentAnnotationCache.put(str, coreDocument);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,628 @@
|
||||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package FunctionLayer
|
||||
|
||||
import DataLayer.DataMapper
|
||||
import FunctionLayer.StanfordParser.SentimentAnalyzerTest
|
||||
import edu.mit.jmwe.data.IMWE
|
||||
import edu.mit.jmwe.data.IToken
|
||||
import edu.stanford.nlp.ie.AbstractSequenceClassifier
|
||||
import edu.stanford.nlp.ie.crf.CRFClassifier
|
||||
import edu.stanford.nlp.ling.CoreAnnotations
|
||||
import edu.stanford.nlp.ling.CoreLabel
|
||||
import edu.stanford.nlp.ling.TaggedWord
|
||||
import edu.stanford.nlp.parser.lexparser.LexicalizedParser
|
||||
import edu.stanford.nlp.pipeline.Annotation
|
||||
import edu.stanford.nlp.pipeline.CoreDocument
|
||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP
|
||||
import edu.stanford.nlp.tagger.maxent.MaxentTagger
|
||||
import edu.stanford.nlp.trees.*
|
||||
import edu.stanford.nlp.util.CoreMap
|
||||
import kotlinx.coroutines.*
|
||||
import org.ejml.simple.SimpleMatrix
|
||||
import java.util.*
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.regex.Pattern
|
||||
import kotlin.collections.ArrayList
|
||||
import kotlin.collections.HashMap
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @author install1
|
||||
*/
|
||||
public class DatahandlerKotlinObsolete {
|
||||
private var pipelineAnnotationCache: HashMap<String, Annotation>
|
||||
private var pipelineSentimentAnnotationCache = HashMap<String, Annotation>()
|
||||
private var coreDocumentAnnotationCache: HashMap<String, CoreDocument>
|
||||
private var jmweAnnotationCache = HashMap<String, Annotation>()
|
||||
|
||||
//private val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz"
|
||||
private val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz"
|
||||
private var tagger: MaxentTagger = MaxentTagger()
|
||||
private var gsf: GrammaticalStructureFactory
|
||||
private var classifier: AbstractSequenceClassifier<CoreLabel>
|
||||
|
||||
//SentimentAnalyzer Hashmaps
|
||||
private var tokenizeCountingHashMap: HashMap<String, Int> = HashMap()
|
||||
private var taggedWordListHashMap: HashMap<String, List<List<TaggedWord>>> = HashMap()
|
||||
private var retrieveTGWListHashMap: HashMap<String, java.util.ArrayList<String>> =
|
||||
HashMap()
|
||||
private var sentences1HashMap: HashMap<String, List<CoreMap>> = HashMap()
|
||||
private var sentencesSentimentHashMap: HashMap<String, List<CoreMap>> = HashMap()
|
||||
private var trees1HashMap: HashMap<String, java.util.ArrayList<Tree>> = HashMap()
|
||||
private var grammaticalStructureHashMap: HashMap<String, java.util.ArrayList<GrammaticalStructure>> =
|
||||
HashMap()
|
||||
private var typedDependenciesHashMap: HashMap<String, java.util.ArrayList<TypedDependency>> =
|
||||
HashMap()
|
||||
private var rnnCoreAnnotationsPredictedHashMap: HashMap<String, java.util.ArrayList<Int>> = HashMap()
|
||||
private var simpleMatricesHashMap: HashMap<String, java.util.ArrayList<SimpleMatrix>> = HashMap()
|
||||
private var simpleMatricesNodevectorsHashMap: HashMap<String, java.util.ArrayList<SimpleMatrix>> = HashMap()
|
||||
private var listHashMap: HashMap<String, MutableList<Any?>> = HashMap()
|
||||
private var longestHashMap: HashMap<String, Int> = HashMap()
|
||||
private var sentimentHashMap: HashMap<String, Int> = HashMap()
|
||||
private var imwesHashMap: HashMap<String, List<IMWE<IToken>>> = HashMap()
|
||||
private var InflectedCounterNegativeHashMap: HashMap<String, Int> = HashMap()
|
||||
private var InflectedCounterPositiveHashMap: HashMap<String, Int> = HashMap()
|
||||
private var tokenEntryHashMap: HashMap<String, ArrayList<String>> = HashMap()
|
||||
private var MarkedContinuousCounterHashMap: HashMap<String, Int> = HashMap()
|
||||
private var UnmarkedPatternCounterHashMap: HashMap<String, Int> = HashMap()
|
||||
private var strTokensIpartFormHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||
private var tokenFormsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||
private var strTokenEntryGetPOSHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||
private var intTokenEntyCountsHashMap: HashMap<String, java.util.ArrayList<Int>> = HashMap()
|
||||
private var ITokenTagsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||
private var strTokenStemsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||
private var AnotatorcounterHashMap: HashMap<String, Int> = HashMap()
|
||||
private var TokensCounterHashMap: HashMap<String, Int> = HashMap()
|
||||
private var entityTokenTagsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||
private var nerEntitiesHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||
private var nerEntitiesTypeHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||
private var stopWordTokenHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||
private var stopWordLemmaHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
|
||||
private var PairCounterHashMap: HashMap<String, Int> = HashMap()
|
||||
|
||||
constructor() {
|
||||
jmweAnnotationCache = HashMap<String, Annotation>()
|
||||
pipelineAnnotationCache = HashMap<String, Annotation>()
|
||||
pipelineSentimentAnnotationCache = HashMap<String, Annotation>()
|
||||
coreDocumentAnnotationCache = HashMap<String, CoreDocument>()
|
||||
gsf = initiateGrammaticalStructureFactory()
|
||||
classifier = CRFClassifier.getClassifierNoExceptions(nerModel)
|
||||
}
|
||||
|
||||
fun initiateGrammaticalStructureFactory(): GrammaticalStructureFactory {
|
||||
val options = arrayOf("-maxLength", "100")
|
||||
//val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
|
||||
val lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"
|
||||
val lp = LexicalizedParser.loadModel(lexParserEnglishPCFG, *options)
|
||||
val tlp = lp.getOp().langpack()
|
||||
return tlp.grammaticalStructureFactory()
|
||||
}
|
||||
|
||||
public fun pipeLineSetUp(): StanfordCoreNLP {
|
||||
val props = Properties()
|
||||
val shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz"
|
||||
//val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz"
|
||||
val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz"
|
||||
//val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz"
|
||||
val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.distsim.crf.ser.gz"
|
||||
props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse")
|
||||
props.setProperty("parse.model", shiftReduceParserPath)
|
||||
props.setProperty("parse.maxlen", "90")
|
||||
props.setProperty("parse.binaryTrees", "true")
|
||||
props.setProperty("threads", "5")
|
||||
props.setProperty("pos.maxlen", "90")
|
||||
props.setProperty("tokenize.maxlen", "90")
|
||||
props.setProperty("ssplit.maxlen", "90")
|
||||
props.setProperty("lemma.maxlen", "90")
|
||||
props.setProperty("ner.model", "$nerModel,$nerModel2,$nerModel3")
|
||||
props.setProperty("ner.combinationMode", "HIGH_RECALL")
|
||||
props.setProperty("regexner.ignorecase", "true")
|
||||
props.setProperty("ner.fine.regexner.ignorecase", "true")
|
||||
props.setProperty("tokenize.options", "untokenizable=firstKeep")
|
||||
return StanfordCoreNLP(props)
|
||||
}
|
||||
|
||||
fun shiftReduceParserInitiate(): StanfordCoreNLP {
|
||||
val propsSentiment = Properties()
|
||||
//val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
|
||||
val lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"
|
||||
val sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz"
|
||||
//val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger"
|
||||
val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words-distsim.tagger"
|
||||
val customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of," +
|
||||
"on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"
|
||||
propsSentiment.setProperty("parse.model", lexParserEnglishPCFG)
|
||||
propsSentiment.setProperty("sentiment.model", sentimentModel)
|
||||
propsSentiment.setProperty("parse.maxlen", "90")
|
||||
propsSentiment.setProperty("threads", "5")
|
||||
propsSentiment.setProperty("pos.maxlen", "90")
|
||||
propsSentiment.setProperty("tokenize.maxlen", "90")
|
||||
propsSentiment.setProperty("ssplit.maxlen", "90")
|
||||
propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword") //coref too expensive memorywise
|
||||
propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator")
|
||||
propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList)
|
||||
propsSentiment.setProperty("tokenize.options", "untokenizable=firstKeep")
|
||||
tagger = MaxentTagger(taggerPath)
|
||||
|
||||
println("finished shiftReduceParserInitiate\n")
|
||||
return StanfordCoreNLP(propsSentiment)
|
||||
}
|
||||
|
||||
private fun trimString(str: String): String {
|
||||
var message = str.trim { it <= ' ' }
|
||||
if (message.startsWith("<@")) {
|
||||
message = message.substring(message.indexOf("> ") + 2)
|
||||
}
|
||||
if (!message.isEmpty()) {
|
||||
message = message.replace("@", "")
|
||||
if (message.contains("<>")) {
|
||||
message = message.substring(message.indexOf(">"))
|
||||
}
|
||||
if (message.startsWith("[ *")) {
|
||||
message = message.substring(message.indexOf("]"))
|
||||
}
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
private fun createStrAnnotation(str: String, stanfordCoreNLP: StanfordCoreNLP, sentimentBool: Boolean) {
|
||||
val strAnno2 = Annotation(str)
|
||||
strAnno2.compact()
|
||||
stanfordCoreNLP.annotate(strAnno2)
|
||||
if (sentimentBool) {
|
||||
pipelineSentimentAnnotationCache.put(str, strAnno2)
|
||||
} else {
|
||||
pipelineAnnotationCache.put(str, strAnno2)
|
||||
}
|
||||
}
|
||||
|
||||
private fun getResponseFutures(strF: String, stanfordCoreNLP: StanfordCoreNLP, stanfordCoreNLPSentiment: StanfordCoreNLP): String {
|
||||
val strAnno: Annotation = Annotation(strF)
|
||||
strAnno.compact()
|
||||
stanfordCoreNLP.annotate(strAnno)
|
||||
|
||||
val strAnnoSentiment: Annotation = Annotation(strF)
|
||||
strAnnoSentiment.compact()
|
||||
stanfordCoreNLPSentiment.annotate(strAnnoSentiment)
|
||||
|
||||
|
||||
val annotation = Annotation(strF)
|
||||
stanfordCoreNLP.annotate(annotation)
|
||||
val coreDocument = CoreDocument(annotation)
|
||||
|
||||
val values_copy: List<String> = ArrayList(DataMapper.getAllStrings())
|
||||
var preRelationUserCounters = -155000.0
|
||||
val concurrentRelations: MutableList<String> = arrayListOf()
|
||||
val SB = StringBuilder()
|
||||
var jmweAnnotationF = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strF)
|
||||
var tokenizeCountingF: Int? = null
|
||||
var taggedWordListF: List<List<TaggedWord>>? = null
|
||||
var retrieveTGWListF: java.util.ArrayList<String>? = null
|
||||
var sentencesF: List<CoreMap>? = null
|
||||
var sentencesSentimentF: List<CoreMap>? = null
|
||||
var coreMaps1: List<CoreMap> = jmweAnnotationF.get(CoreAnnotations.SentencesAnnotation::class.java)
|
||||
var treesF: java.util.ArrayList<Tree>? = null
|
||||
var grammaticalStructuresF: ArrayList<GrammaticalStructure>? = null
|
||||
var typedDependenciesF: java.util.ArrayList<TypedDependency>? = null
|
||||
var rnnCoreAnnotationsPredictedF: java.util.ArrayList<Int>? = null
|
||||
var simpleMatricesF: java.util.ArrayList<SimpleMatrix>? = null
|
||||
var simpleMatricesNodevectorsF: java.util.ArrayList<SimpleMatrix>? = null
|
||||
var listF: MutableList<Any?>? = null
|
||||
var longestF: Int? = null
|
||||
var sentimentLongestF: Int? = null
|
||||
var imwesF: List<IMWE<IToken>>? = null
|
||||
var InflectedCounterNegativeF: Int? = null
|
||||
var InflectedCounterPositiveF: Int? = null
|
||||
var tokenEntryF: ArrayList<String>? = null
|
||||
var MarkedContinuousCounterF: Int? = null
|
||||
var UnmarkedPatternCounterF: Int? = null
|
||||
var strTokensIpartFormF: ArrayList<String>? = null
|
||||
var tokenFormsF: java.util.ArrayList<String>? = null
|
||||
var strTokenEntryGetPOSF: ArrayList<String>? = null
|
||||
var intTokenEntyCountsF: java.util.ArrayList<Int>? = null
|
||||
var ITokenTagsF: ArrayList<String>? = null
|
||||
var strTokenStemsF: java.util.ArrayList<String>? = null
|
||||
var AnotatorcounterF: Int? = null
|
||||
var TokensCounterF: Int? = null
|
||||
var entityTokenTagsF: java.util.ArrayList<String>? = null
|
||||
var nerEntitiesF: java.util.ArrayList<String>? = null
|
||||
var nerEntitiesTypeF: java.util.ArrayList<String>? = null
|
||||
var stopWordTokenF: java.util.ArrayList<String>? = null
|
||||
var stopWordLemmaF: java.util.ArrayList<String>? = null
|
||||
var PairCounterF: Int? = null
|
||||
for (str1 in values_copy) {
|
||||
if (strF != str1) {
|
||||
val annotation2 = pipelineSentimentAnnotationCache.getOrDefault(str1, null)
|
||||
val annotation4 = pipelineAnnotationCache.getOrDefault(str1, null)
|
||||
val coreDocument1 = coreDocumentAnnotationCache.getOrDefault(str1, null)
|
||||
var jmweAnnotation = jmweAnnotationCache.getOrDefault(str1, null)
|
||||
if (annotation2 == null) {
|
||||
createStrAnnotation(str1, stanfordCoreNLPSentiment, true)
|
||||
}
|
||||
if (annotation4 == null) {
|
||||
createStrAnnotation(str1, stanfordCoreNLP, false)
|
||||
}
|
||||
if (coreDocument1 == null) {
|
||||
getCoreDocumentsSuggested(stanfordCoreNLP, str1)
|
||||
}
|
||||
if (jmweAnnotation == null) {
|
||||
getJMWEAnnotation(str1)
|
||||
jmweAnnotation = jmweAnnotationCache.get(str1)
|
||||
}
|
||||
val tokenizeCounting: Int? = tokenizeCountingHashMap.getOrDefault(str1, null)
|
||||
val taggedWordList1: List<List<TaggedWord>>? = taggedWordListHashMap.getOrDefault(str1, null)
|
||||
val retrieveTGWList1: java.util.ArrayList<String>? = retrieveTGWListHashMap.getOrDefault(str1, null)
|
||||
val sentence1: List<CoreMap>? = sentences1HashMap.getOrDefault(str1, null)
|
||||
val sentenceSentiment1: List<CoreMap>? = sentencesSentimentHashMap.getOrDefault(str1, null)
|
||||
val trees1 = trees1HashMap.getOrDefault(str1, null)
|
||||
var coreMaps2: List<CoreMap> = listOf()
|
||||
val grammaticalStructures1 = grammaticalStructureHashMap.getOrDefault(
|
||||
str1, null)
|
||||
if (jmweAnnotation != null) {
|
||||
coreMaps2 = jmweAnnotation.get(CoreAnnotations.SentencesAnnotation::class.java)
|
||||
}
|
||||
val typedDependencies1 = typedDependenciesHashMap.getOrDefault(str1, null)
|
||||
val rnnCoreAnnotationsPredicted1 = rnnCoreAnnotationsPredictedHashMap.getOrDefault(str1, null)
|
||||
val simpleMatrices1 = simpleMatricesHashMap.getOrDefault(str1, null);
|
||||
val simpleMatricesNodevectors1 = simpleMatricesNodevectorsHashMap.getOrDefault(str1, null);
|
||||
val list1 = listHashMap.getOrDefault(str1, null);
|
||||
val longest1 = longestHashMap.getOrDefault(str1, null);
|
||||
val sentimentLongest1 = sentimentHashMap.getOrDefault(str1, null);
|
||||
val imwes1 = imwesHashMap.getOrDefault(str1, null);
|
||||
val InflectedCounterNegative1 = InflectedCounterNegativeHashMap.getOrDefault(str1, null);
|
||||
val InflectedCounterPositive1 = InflectedCounterPositiveHashMap.getOrDefault(str1, null)
|
||||
val tokenEntry1 = tokenEntryHashMap.getOrDefault(str1, null)
|
||||
val MarkedContinuousCounter1 = MarkedContinuousCounterHashMap.getOrDefault(str1, null)
|
||||
val UnmarkedPatternCounter1 = UnmarkedPatternCounterHashMap.getOrDefault(str1, null)
|
||||
val strTokensIpartForm1 = strTokensIpartFormHashMap.getOrDefault(str1, null);
|
||||
val tokenForms1 = tokenFormsHashMap.getOrDefault(str1, null);
|
||||
val strTokenEntryGetPOS1 = strTokenEntryGetPOSHashMap.getOrDefault(str1, null)
|
||||
val intTokenEntyCounts1 = intTokenEntyCountsHashMap.getOrDefault(str1, null);
|
||||
val ITokenTags1 = ITokenTagsHashMap.getOrDefault(str1, null);
|
||||
val strTokenStems1 = strTokenStemsHashMap.getOrDefault(str1, null);
|
||||
val Anotatorcounter1 = AnotatorcounterHashMap.getOrDefault(str1, null);
|
||||
val TokensCounter1 = TokensCounterHashMap.getOrDefault(str1, null);
|
||||
val entityTokenTags1 = entityTokenTagsHashMap.getOrDefault(str1, null);
|
||||
val nerEntities1 = nerEntitiesHashMap.getOrDefault(str1, null);
|
||||
val nerEntitiesType1 = nerEntitiesTypeHashMap.getOrDefault(str1, null);
|
||||
val stopWordToken1 = stopWordTokenHashMap.getOrDefault(str1, null);
|
||||
val stopWordLemma1 = stopWordLemmaHashMap.getOrDefault(str1, null);
|
||||
val PairCounter1 = PairCounterHashMap.getOrDefault(str1, null);
|
||||
|
||||
var SMX = SentimentAnalyzerTest(strF, str1, SimilarityMatrix(strF, str1),
|
||||
coreMaps1, coreMaps2, strAnno,
|
||||
pipelineAnnotationCache[str1], strAnnoSentiment,
|
||||
pipelineSentimentAnnotationCache[str1], coreDocument, coreDocumentAnnotationCache[str1],
|
||||
tagger, gsf, classifier, tokenizeCounting, tokenizeCountingF,
|
||||
taggedWordListF, taggedWordList1, retrieveTGWListF, retrieveTGWList1,
|
||||
sentencesF, sentence1, sentencesSentimentF, sentenceSentiment1, treesF, trees1,
|
||||
grammaticalStructuresF, grammaticalStructures1, typedDependenciesF,
|
||||
typedDependencies1, rnnCoreAnnotationsPredictedF, rnnCoreAnnotationsPredicted1,
|
||||
simpleMatricesF, simpleMatrices1, simpleMatricesNodevectorsF, simpleMatricesNodevectors1,
|
||||
listF, list1, longestF, longest1, sentimentLongestF, sentimentLongest1, imwesF,
|
||||
imwes1, InflectedCounterNegativeF, InflectedCounterNegative1, InflectedCounterPositiveF,
|
||||
InflectedCounterPositive1, tokenEntryF, tokenEntry1, MarkedContinuousCounterF,
|
||||
MarkedContinuousCounter1, UnmarkedPatternCounterF, UnmarkedPatternCounter1,
|
||||
strTokensIpartFormF, strTokensIpartForm1, tokenFormsF, tokenForms1,
|
||||
strTokenEntryGetPOSF, strTokenEntryGetPOS1, intTokenEntyCountsF,
|
||||
intTokenEntyCounts1, ITokenTagsF, ITokenTags1, strTokenStemsF, strTokenStems1,
|
||||
AnotatorcounterF, Anotatorcounter1, TokensCounterF, TokensCounter1,
|
||||
entityTokenTagsF, entityTokenTags1, nerEntitiesF, nerEntities1, nerEntitiesTypeF,
|
||||
nerEntitiesType1, stopWordTokenF, stopWordToken1, stopWordLemmaF, stopWordLemma1,
|
||||
PairCounterF, PairCounter1)
|
||||
if (tokenizeCounting == null) {
|
||||
tokenizeCountingHashMap.put(str1, SMX.getTokenizeCounting())
|
||||
}
|
||||
if (taggedWordList1 == null) {
|
||||
taggedWordListHashMap.put(str1, SMX.getTaggedWordList1())
|
||||
}
|
||||
if (tokenizeCountingF == null) {
|
||||
tokenizeCountingF = SMX.getTokenizeCountingF();
|
||||
}
|
||||
if (taggedWordListF == null) {
|
||||
taggedWordListF = SMX.getTaggedWordListF();
|
||||
}
|
||||
if (retrieveTGWListF == null) {
|
||||
retrieveTGWListF = SMX.getRetrieveTGWListF();
|
||||
}
|
||||
if (retrieveTGWList1 == null) {
|
||||
retrieveTGWListHashMap.put(str1, SMX.getRetrieveTGWList1());
|
||||
}
|
||||
if (sentencesF == null) {
|
||||
sentencesF = SMX.getSentencesF();
|
||||
}
|
||||
if (sentence1 == null) {
|
||||
sentences1HashMap.put(str1, SMX.getSentences1())
|
||||
}
|
||||
if (sentencesSentimentF == null) {
|
||||
sentencesSentimentF = SMX.getSentencesSentimentF();
|
||||
}
|
||||
if (sentenceSentiment1 == null) {
|
||||
sentencesSentimentHashMap.put(str1, SMX.getSentencesSentiment1());
|
||||
}
|
||||
if (treesF == null) {
|
||||
treesF = SMX.getTreesF();
|
||||
}
|
||||
if (trees1 == null) {
|
||||
trees1HashMap.put(str1, SMX.getTrees1())
|
||||
}
|
||||
if (grammaticalStructuresF == null) {
|
||||
grammaticalStructuresF = SMX.getGrammaticalStructuresF();
|
||||
}
|
||||
if (grammaticalStructures1 == null) {
|
||||
grammaticalStructureHashMap.put(str1, SMX.getGrammaticalStructures1())
|
||||
}
|
||||
if (typedDependenciesF == null) {
|
||||
typedDependenciesF = SMX.getTypedDependenciesF();
|
||||
}
|
||||
if (typedDependencies1 == null) {
|
||||
typedDependenciesHashMap.put(str1, SMX.getTypedDependencies1())
|
||||
}
|
||||
if (rnnCoreAnnotationsPredictedF == null) {
|
||||
rnnCoreAnnotationsPredictedF = SMX.getRnnCoreAnnotationsPredictedF()
|
||||
}
|
||||
if (rnnCoreAnnotationsPredicted1 == null) {
|
||||
rnnCoreAnnotationsPredictedHashMap.put(str1, SMX.getRnnCoreAnnotationsPredicted1())
|
||||
}
|
||||
if (simpleMatricesF == null) {
|
||||
simpleMatricesF = SMX.getSimpleMatricesF();
|
||||
}
|
||||
if (simpleMatrices1 == null) {
|
||||
simpleMatricesHashMap.put(str1, SMX.getSimpleMatrices1());
|
||||
}
|
||||
if (simpleMatricesNodevectorsF == null) {
|
||||
simpleMatricesNodevectorsF = SMX.getSimpleMatricesNodevectorsF();
|
||||
}
|
||||
if (simpleMatricesNodevectors1 == null) {
|
||||
simpleMatricesNodevectorsHashMap.put(str1, SMX.getSimpleMatricesNodevectors1());
|
||||
}
|
||||
if (listF == null) {
|
||||
listF = SMX.getListF();
|
||||
}
|
||||
if (list1 == null) {
|
||||
listHashMap.put(str1, SMX.getList1());
|
||||
}
|
||||
if (longestF == null) {
|
||||
longestF = SMX.getLongestF();
|
||||
}
|
||||
if (longest1 == null) {
|
||||
longestHashMap.put(str1, SMX.getLongest1());
|
||||
}
|
||||
if (sentimentLongestF == null) {
|
||||
sentimentLongestF = SMX.getSentimentLongestF();
|
||||
}
|
||||
if (sentimentLongest1 == null) {
|
||||
sentimentHashMap.put(str1, SMX.getSentimentLongest1());
|
||||
}
|
||||
if (imwesF == null) {
|
||||
imwesF = SMX.getImwesF();
|
||||
}
|
||||
if (imwes1 == null) {
|
||||
imwesHashMap.put(str1, SMX.getImwes1());
|
||||
}
|
||||
if (InflectedCounterNegativeF == null) {
|
||||
InflectedCounterNegativeF = SMX.getInflectedCounterNegativeF();
|
||||
}
|
||||
if (InflectedCounterNegative1 == null) {
|
||||
InflectedCounterNegativeHashMap.put(str1, SMX.getInflectedCounterNegative1());
|
||||
}
|
||||
if (InflectedCounterPositiveF == null) {
|
||||
InflectedCounterPositiveF = SMX.getInflectedCounterPositiveF();
|
||||
}
|
||||
if (InflectedCounterPositive1 == null) {
|
||||
InflectedCounterPositiveHashMap.put(str1, SMX.getInflectedCounterPositive1());
|
||||
}
|
||||
if (tokenEntryF == null) {
|
||||
tokenEntryF = SMX.getTokenEntryF();
|
||||
}
|
||||
if (tokenEntry1 == null) {
|
||||
tokenEntryHashMap.put(str1, SMX.getTokenEntry1())
|
||||
}
|
||||
if (MarkedContinuousCounterF == null) {
|
||||
MarkedContinuousCounterF = SMX.getMarkedContinuousCounterF();
|
||||
}
|
||||
if (MarkedContinuousCounter1 == null) {
|
||||
MarkedContinuousCounterHashMap.put(str1, SMX.getMarkedContinuousCounter1());
|
||||
}
|
||||
if (UnmarkedPatternCounterF == null) {
|
||||
UnmarkedPatternCounterF = SMX.getUnmarkedPatternCounterF();
|
||||
}
|
||||
if (UnmarkedPatternCounter1 == null) {
|
||||
UnmarkedPatternCounterHashMap.put(str1, SMX.getUnmarkedPatternCounter1());
|
||||
}
|
||||
if (strTokensIpartFormF == null) {
|
||||
strTokensIpartFormF = SMX.getStrTokensIpartFormF();
|
||||
}
|
||||
if (strTokensIpartForm1 == null) {
|
||||
strTokensIpartFormHashMap.put(str1, SMX.getStrTokensIpartForm1());
|
||||
}
|
||||
if (tokenFormsF == null) {
|
||||
tokenFormsF = SMX.getTokenFormsF();
|
||||
}
|
||||
if (tokenForms1 == null) {
|
||||
tokenFormsHashMap.put(str1, SMX.getTokenForms1());
|
||||
}
|
||||
if (strTokenEntryGetPOSF == null) {
|
||||
strTokenEntryGetPOSF = SMX.getStrTokenEntryGetPOSF();
|
||||
}
|
||||
if (strTokenEntryGetPOS1 == null) {
|
||||
strTokenEntryGetPOSHashMap.put(str1, SMX.getStrTokenEntryGetPOS1())
|
||||
}
|
||||
if (intTokenEntyCountsF == null) {
|
||||
intTokenEntyCountsF = SMX.getIntTokenEntyCountsF();
|
||||
}
|
||||
if (intTokenEntyCounts1 == null) {
|
||||
intTokenEntyCountsHashMap.put(str1, SMX.getIntTokenEntyCounts1());
|
||||
}
|
||||
if (ITokenTagsF == null) {
|
||||
ITokenTagsF = SMX.getITokenTagsF();
|
||||
}
|
||||
if (ITokenTags1 == null) {
|
||||
ITokenTagsHashMap.put(str1, SMX.getITokenTags1());
|
||||
}
|
||||
if (strTokenStemsF == null) {
|
||||
strTokenStemsF = SMX.getStrTokenStemsF();
|
||||
}
|
||||
if (strTokenStems1 == null) {
|
||||
strTokenStemsHashMap.put(str1, SMX.getStrTokenStems1());
|
||||
}
|
||||
if (AnotatorcounterF == null) {
|
||||
AnotatorcounterF = SMX.getAnotatorcounterF();
|
||||
}
|
||||
if (Anotatorcounter1 == null) {
|
||||
AnotatorcounterHashMap.put(str1, SMX.getAnotatorcounter1());
|
||||
}
|
||||
if (TokensCounterF == null) {
|
||||
TokensCounterF = SMX.getTokensCounterF();
|
||||
}
|
||||
if (TokensCounter1 == null) {
|
||||
TokensCounterHashMap.put(str1, SMX.getTokensCounter1());
|
||||
}
|
||||
if (entityTokenTagsF == null) {
|
||||
entityTokenTagsF = SMX.getEntityTokenTagsF();
|
||||
}
|
||||
if (entityTokenTags1 == null) {
|
||||
entityTokenTagsHashMap.put(str1, SMX.getEntityTokenTags1());
|
||||
}
|
||||
if (nerEntitiesF == null) {
|
||||
nerEntitiesF = SMX.getNerEntitiesF();
|
||||
}
|
||||
if (nerEntities1 == null) {
|
||||
nerEntitiesHashMap.put(str1, SMX.getNerEntities1());
|
||||
}
|
||||
if (nerEntitiesTypeF == null) {
|
||||
nerEntitiesTypeF = SMX.getNerEntitiesTypeF();
|
||||
}
|
||||
if (nerEntitiesType1 == null) {
|
||||
nerEntitiesTypeHashMap.put(str1, SMX.getNerEntitiesType1());
|
||||
}
|
||||
if (stopWordTokenF == null) {
|
||||
stopWordTokenF = SMX.getStopWordTokenF();
|
||||
}
|
||||
if (stopWordToken1 == null) {
|
||||
stopWordTokenHashMap.put(str1, SMX.getStopWordToken1());
|
||||
}
|
||||
if (stopWordLemmaF == null) {
|
||||
stopWordLemmaF = SMX.getStopWordLemmaF();
|
||||
}
|
||||
if (stopWordLemma1 == null) {
|
||||
stopWordLemmaHashMap.put(str1, SMX.getStopWordLemma1());
|
||||
}
|
||||
if (PairCounterF == null) {
|
||||
PairCounterF = SMX.getPairCounterF();
|
||||
}
|
||||
if (PairCounter1 == null) {
|
||||
PairCounterHashMap.put(str1, SMX.getPairCounter1());
|
||||
}
|
||||
|
||||
var getSMX: SimilarityMatrix = SMX.callSMX()
|
||||
val scoreRelationLastUserMsg = getSMX.distance
|
||||
if (scoreRelationLastUserMsg > preRelationUserCounters) {
|
||||
preRelationUserCounters = scoreRelationLastUserMsg
|
||||
concurrentRelations.add(getSMX.secondaryString)
|
||||
}
|
||||
}
|
||||
}
|
||||
val cacheRequirement = 6500;
|
||||
if (preRelationUserCounters > cacheRequirement && !values_copy.contains(strF) && filterContent(strF)) {
|
||||
DataMapper.InsertMYSQLStrings(strF)
|
||||
DataMapper.checkStringsToDelete();
|
||||
}
|
||||
val randomLenghtPermit = strF.length * (Math.random() * Math.random() * Math.random() * (Math.random() * 10))
|
||||
Collections.reverse(concurrentRelations)
|
||||
val mysqlUpdateLastUsed: ArrayList<String> = ArrayList()
|
||||
if (!concurrentRelations.isEmpty()) {
|
||||
for (secondaryRelation in concurrentRelations) {
|
||||
if (SB.toString().length > randomLenghtPermit && !SB.toString().isEmpty()) {
|
||||
break
|
||||
}
|
||||
SB.append(secondaryRelation).append(" ")
|
||||
mysqlUpdateLastUsed.add(secondaryRelation)
|
||||
}
|
||||
}
|
||||
if (SB.toString().isEmpty()) {
|
||||
return "failure, preventing stuckness"
|
||||
}
|
||||
DataMapper.updateLastUsed(mysqlUpdateLastUsed);
|
||||
return SB.toString()
|
||||
}
|
||||
|
||||
private fun getJMWEAnnotation(str1: String) {
|
||||
val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str1)
|
||||
jmweAnnotationCache.put(str1, jmweAnnotation)
|
||||
}
|
||||
|
||||
fun getResponseMsg(str: String, personName: String, stanfordCoreNLP: StanfordCoreNLP,
|
||||
stanfordCoreNLPSentiment: StanfordCoreNLP, ingameResponse: Boolean): String {
|
||||
var responseFutures: String = ""
|
||||
runBlocking {
|
||||
launch(Dispatchers.Default) {
|
||||
var strF = trimString(str)
|
||||
responseFutures = getResponseFutures(strF, stanfordCoreNLP, stanfordCoreNLPSentiment)
|
||||
if (!ingameResponse) {
|
||||
responseFutures = checkPersonPresentInSentence(personName, responseFutures, strF, stanfordCoreNLP,
|
||||
stanfordCoreNLPSentiment)
|
||||
}
|
||||
yield()
|
||||
}.join()
|
||||
}
|
||||
return responseFutures
|
||||
}
|
||||
|
||||
private fun checkPersonPresentInSentence(personName: String, responseMsg: String, userLastMessage: String,
|
||||
stanfordCoreNLP: StanfordCoreNLP,
|
||||
stanfordCoreNLPSentiment: StanfordCoreNLP): String {
|
||||
try {
|
||||
val pipelineCoreDcoument = CoreDocument(responseMsg)
|
||||
val pipelineCoreDcoumentLastMsg = CoreDocument(userLastMessage)
|
||||
stanfordCoreNLP.annotate(pipelineCoreDcoument)
|
||||
stanfordCoreNLPSentiment.annotate(pipelineCoreDcoumentLastMsg)
|
||||
val regex = "(.*?\\d){10,}"
|
||||
for (em in pipelineCoreDcoument.entityMentions()) {
|
||||
val entityType = em.entityType()
|
||||
if (entityType == "PERSON") {
|
||||
var str = responseMsg
|
||||
val emText = em.text()
|
||||
val pattern = Pattern.compile(regex)
|
||||
val matcher = pattern.matcher(personName)
|
||||
val isMatched = matcher.matches()
|
||||
if (emText != personName && !isMatched) {
|
||||
for (emLastMsg in pipelineCoreDcoumentLastMsg.entityMentions()) {
|
||||
if (emText != emLastMsg.text() && !Character.isDigit(emLastMsg.text().trim { it <= ' ' }[0])) {
|
||||
str = (responseMsg.substring(0, responseMsg.indexOf(emText)) + " "
|
||||
+ emLastMsg + " " + responseMsg.substring(responseMsg.indexOf(emText)))
|
||||
}
|
||||
}
|
||||
str += " $personName"
|
||||
return str
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
println("""SCUFFED JAYZ: ${e.message}""".trimIndent())
|
||||
}
|
||||
return responseMsg
|
||||
}
|
||||
|
||||
fun filterContent(str: String): Boolean {
|
||||
if (!str.isEmpty() && str.length > 3) {
|
||||
var str1Local: String = str.trim();
|
||||
if (str1Local.length > 2 && !str1Local.startsWith("!")) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
fun getCoreDocumentsSuggested(pipeline: StanfordCoreNLP, str: String) {
|
||||
val annotation = Annotation(str)
|
||||
pipeline.annotate(annotation)
|
||||
val coreDocument = CoreDocument(annotation)
|
||||
coreDocumentAnnotationCache.put(str, coreDocument)
|
||||
}
|
||||
}
|
||||
@ -5,20 +5,13 @@
|
||||
*/
|
||||
package FunctionLayer;
|
||||
|
||||
import PresentationLayer.DiscordHandler;
|
||||
import discord4j.core.event.domain.message.MessageCreateEvent;
|
||||
import discord4j.core.object.entity.User;
|
||||
import discord4j.core.object.entity.channel.TextChannel;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
|
||||
/**
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import FunctionLayer.Datahandler;
|
||||
import FunctionLayer.DatahandlerKotlinObsolete;
|
||||
import FunctionLayer.PipelineJMWESingleton;
|
||||
import FunctionLayer.StanfordParser.SentimentAnalyzerTest;
|
||||
import edu.mit.jmwe.data.IMWE;
|
||||
@ -17,10 +17,9 @@ import edu.stanford.nlp.trees.*;
|
||||
import edu.stanford.nlp.util.CoreMap;
|
||||
import org.ejml.simple.SimpleMatrix;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import FunctionLayer.SimilarityMatrix;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@ -178,7 +177,7 @@ public class junit {
|
||||
|
||||
//@Test
|
||||
public void testScoring() {
|
||||
Datahandler datahandler = new Datahandler();
|
||||
DatahandlerKotlinObsolete datahandler = new DatahandlerKotlinObsolete();
|
||||
PipelineJMWESingleton.getINSTANCE();
|
||||
StanfordCoreNLP stanfordCoreNLP = datahandler.pipeLineSetUp();
|
||||
StanfordCoreNLP stanfordCoreNLPSentiment = datahandler.shiftReduceParserInitiate();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user