diff --git a/ArtificialAutism/src/main/java/DataLayer/DataMapper.java b/ArtificialAutism/src/main/java/DataLayer/DataMapper.java index 1103b037..9c09a8c4 100644 --- a/ArtificialAutism/src/main/java/DataLayer/DataMapper.java +++ b/ArtificialAutism/src/main/java/DataLayer/DataMapper.java @@ -20,7 +20,7 @@ import java.util.logging.Logger; */ public class DataMapper { - public static ArrayList getAllStrings() throws SQLException { + public static ArrayList getAllStrings() { Connection l_cCon = null; PreparedStatement l_pStatement = null; ResultSet l_rsSearch = null; @@ -33,13 +33,15 @@ public class DataMapper { while (l_rsSearch.next()) { arrayListStr.add(l_rsSearch.getString(1)); } + } catch (SQLException throwables) { + throwables.printStackTrace(); } finally { CloseConnections(l_pStatement, l_rsSearch, l_cCon); } return arrayListStr; } - public static void InsertMYSQLStrings(String str) throws SQLException { + public static void InsertMYSQLStrings(String str) { Connection l_cCon = null; PreparedStatement l_pStatement = null; ResultSet l_rsSearch = null; @@ -49,6 +51,8 @@ public class DataMapper { l_pStatement = l_cCon.prepareStatement(l_sSQL); l_pStatement.setString(1, str); l_pStatement.execute(); + } catch (SQLException throwables) { + throwables.printStackTrace(); } finally { CloseConnections(l_pStatement, l_rsSearch, l_cCon); } diff --git a/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java new file mode 100644 index 00000000..c81c6d89 --- /dev/null +++ b/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java @@ -0,0 +1,677 @@ +package FunctionLayer; + +import DataLayer.DataMapper; +import FunctionLayer.StanfordParser.SentimentAnalyzerTest; +import edu.mit.jmwe.data.IMWE; +import edu.mit.jmwe.data.IToken; +import edu.stanford.nlp.ie.AbstractSequenceClassifier; +import edu.stanford.nlp.ie.crf.CRFClassifier; +import edu.stanford.nlp.ie.machinereading.structure.EntityMention; +import edu.stanford.nlp.ling.CoreAnnotations; +import edu.stanford.nlp.ling.CoreLabel; +import edu.stanford.nlp.ling.TaggedWord; +import edu.stanford.nlp.parser.lexparser.LexicalizedParser; +import edu.stanford.nlp.pipeline.Annotation; +import edu.stanford.nlp.pipeline.CoreDocument; +import edu.stanford.nlp.pipeline.CoreEntityMention; +import edu.stanford.nlp.pipeline.StanfordCoreNLP; +import edu.stanford.nlp.tagger.maxent.MaxentTagger; +import edu.stanford.nlp.trees.*; +import edu.stanford.nlp.util.CoreMap; +import kotlinx.coroutines.*; +import org.ejml.simple.SimpleMatrix; + +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +public class Datahandler { + private HashMap pipelineAnnotationCache; + private HashMap pipelineSentimentAnnotationCache; + private HashMap coreDocumentAnnotationCache; + private HashMap jmweAnnotationCache; + + private MaxentTagger tagger = new MaxentTagger(); + + private GrammaticalStructureFactory gsf; + private AbstractSequenceClassifier classifier; + + //SentimentAnalyzer Hashmaps + private HashMap tokenizeCountingHashMap = new HashMap(); + + private HashMap>> taggedWordListHashMap = new HashMap(); + + private HashMap> retrieveTGWListHashMap = new HashMap(); + + private HashMap> sentences1HashMap = new HashMap(); + + private HashMap> sentencesSentimentHashMap = new HashMap(); + + private HashMap> trees1HashMap = new HashMap(); + + private HashMap> grammaticalStructureHashMap = new HashMap(); + + private HashMap> typedDependenciesHashMap = new HashMap(); + + private HashMap> rnnCoreAnnotationsPredictedHashMap = new HashMap(); + + private HashMap> simpleMatricesHashMap = new HashMap(); + + private HashMap> simpleMatricesNodevectorsHashMap = new HashMap(); + + private HashMap listHashMap = new HashMap(); + + private HashMap longestHashMap = new HashMap(); + + private HashMap sentimentHashMap = new HashMap(); + + private HashMap>> imwesHashMap = new HashMap(); + + private HashMap InflectedCounterNegativeHashMap = new HashMap(); + + private HashMap InflectedCounterPositiveHashMap = new HashMap(); + + private HashMap> tokenEntryHashMap = new HashMap(); + + private HashMap MarkedContinuousCounterHashMap = new HashMap(); + + private HashMap UnmarkedPatternCounterHashMap = new HashMap(); + + private HashMap> strTokensIpartFormHashMap = new HashMap(); + + private HashMap> tokenFormsHashMap = new HashMap(); + + private HashMap> strTokenEntryGetPOSHashMap = new HashMap(); + + private HashMap> intTokenEntyCountsHashMap = new HashMap(); + + private HashMap> ITokenTagsHashMap = new HashMap(); + + private HashMap> strTokenStemsHashMap = new HashMap(); + + private HashMap AnotatorcounterHashMap = new HashMap(); + + private HashMap TokensCounterHashMap = new HashMap(); + + private HashMap> entityTokenTagsHashMap = new HashMap(); + + private HashMap> nerEntitiesHashMap = new HashMap(); + + private HashMap> nerEntitiesTypeHashMap = new HashMap(); + + private HashMap> stopWordTokenHashMap = new HashMap(); + + private HashMap> stopWordLemmaHashMap = new HashMap(); + + private HashMap PairCounterHashMap = new HashMap(); + + public Datahandler() { + jmweAnnotationCache = new HashMap(); + pipelineAnnotationCache = new HashMap(); + pipelineSentimentAnnotationCache = new HashMap(); + coreDocumentAnnotationCache = new HashMap(); + gsf = initiateGrammaticalStructureFactory(); + String nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz"; + classifier = CRFClassifier.getClassifierNoExceptions(nerModel); + } + + private GrammaticalStructureFactory initiateGrammaticalStructureFactory() { + // lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz" + String lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"; + LexicalizedParser lp = LexicalizedParser.loadModel(lexParserEnglishPCFG, "-maxLength", "100"); + TreebankLanguagePack tlp = lp.getOp().langpack(); + return tlp.grammaticalStructureFactory(); + } + + public StanfordCoreNLP pipeLineSetUp() { + Properties props = new Properties(); + String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz"; + // nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz" + //String nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz"; + // nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz" + //String nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.distsim.crf.ser.gz"; + props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse"); + props.setProperty("parse.model", shiftReduceParserPath); + props.setProperty("parse.maxlen", "90"); + props.setProperty("parse.binaryTrees", "true"); + props.setProperty("threads", "5"); + props.setProperty("pos.maxlen", "90"); + props.setProperty("tokenize.maxlen", "90"); + props.setProperty("ssplit.maxlen", "90"); + props.setProperty("lemma.maxlen", "90"); + props.setProperty("ner.model", "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz" + + ",edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz" + + ",edu/stanford/nlp/models/ner/english.muc.7class.distsim.crf.ser.gz"); + props.setProperty("ner.combinationMode", "HIGH_RECALL"); + props.setProperty("regexner.ignorecase", "true"); + props.setProperty("ner.fine.regexner.ignorecase", "true"); + props.setProperty("tokenize.options", "untokenizable=firstKeep"); + return new StanfordCoreNLP(props); + } + + public StanfordCoreNLP shiftReduceParserInitiate() { + Properties propsSentiment = new Properties(); + // lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz" + String lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"; + String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz"; + // taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger" + String taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words-distsim.tagger"; + String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of," + + "on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"; + propsSentiment.setProperty("parse.model", lexParserEnglishPCFG); + propsSentiment.setProperty("sentiment.model", sentimentModel); + propsSentiment.setProperty("parse.maxlen", "90"); + propsSentiment.setProperty("threads", "5"); + propsSentiment.setProperty("pos.maxlen", "90"); + propsSentiment.setProperty("tokenize.maxlen", "90"); + propsSentiment.setProperty("ssplit.maxlen", "90"); + propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword"); //coref too expensive memorywise + propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator"); + propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList); + propsSentiment.setProperty("tokenize.options", "untokenizable=firstKeep"); + tagger = new MaxentTagger(taggerPath); + return new StanfordCoreNLP(propsSentiment); + } + + private String trimString(String str) { + String message = str.trim(); + if (message.startsWith("<@")) { + message = message.substring(message.indexOf("> ") + 2); + } + if (!message.isEmpty()) { + message = message.replace("@", ""); + if (message.contains("<>")) { + message = message.substring(message.indexOf(">")); + } + if (message.startsWith("[ *")) { + message = message.substring(message.indexOf("]")); + } + } + return message; + } + + private void createStrAnnotation(String str, StanfordCoreNLP stanfordCoreNLP, Boolean sentimentBool) { + Annotation strAnno2 = new Annotation(str); + strAnno2.compact(); + stanfordCoreNLP.annotate(strAnno2); + if (sentimentBool) { + pipelineSentimentAnnotationCache.put(str, strAnno2); + } else { + pipelineAnnotationCache.put(str, strAnno2); + } + } + + private String getResponseFutures(String strF, StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) { + + Annotation strAnno = new Annotation(strF); + strAnno.compact(); + stanfordCoreNLP.annotate(strAnno); + + Annotation strAnnoSentiment = new Annotation(strF); + strAnnoSentiment.compact(); + stanfordCoreNLPSentiment.annotate(strAnnoSentiment); + + Annotation annotation = new Annotation(strF); + stanfordCoreNLP.annotate(annotation); + CoreDocument coreDocument = new CoreDocument(annotation); + + List ues_copy = new ArrayList(DataMapper.getAllStrings()); + double preRelationUserCounters = -155000.0; + ArrayList concurrentRelations = new ArrayList(); + StringBuilder SB = new StringBuilder(); + Annotation jmweAnnotationF = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strF); + Integer tokenizeCountingF = null; + List> taggedWordListF = null; + java.util.ArrayList retrieveTGWListF = null; + List sentencesF = null; + List sentencesSentimentF = null; + + List coreMaps1 = jmweAnnotationF.get(CoreAnnotations.SentencesAnnotation.class); + + java.util.ArrayList treesF = null; + + ArrayList grammaticalStructuresF = null; + + java.util.ArrayList typedDependenciesF = null; + + java.util.ArrayList rnnCoreAnnotationsPredictedF = null; + + java.util.ArrayList simpleMatricesF = null; + + java.util.ArrayList simpleMatricesNodevectorsF = null; + List listF = null; + + Integer longestF = null; + + Integer sentimentLongestF = null; + + List> imwesF = null; + + Integer InflectedCounterNegativeF = null; + + Integer InflectedCounterPositiveF = null; + + ArrayList tokenEntryF = null; + + Integer MarkedContinuousCounterF = null; + + Integer UnmarkedPatternCounterF = null; + + ArrayList strTokensIpartFormF = null; + + java.util.ArrayList tokenFormsF = null; + + ArrayList strTokenEntryGetPOSF = null; + + java.util.ArrayList intTokenEntyCountsF = null; + + ArrayList ITokenTagsF = null; + + java.util.ArrayList strTokenStemsF = null; + + Integer AnotatorcounterF = null; + + Integer TokensCounterF = null; + + java.util.ArrayList entityTokenTagsF = null; + + java.util.ArrayList nerEntitiesF = null; + + java.util.ArrayList nerEntitiesTypeF = null; + + java.util.ArrayList stopWordTokenF = null; + + java.util.ArrayList stopWordLemmaF = null; + + Integer PairCounterF = null; + for (String str1 : ues_copy) { + if (strF != str1) { + Annotation annotation2 = pipelineSentimentAnnotationCache.getOrDefault(str1, null); + Annotation annotation4 = pipelineAnnotationCache.getOrDefault(str1, null); + CoreDocument coreDocument1 = coreDocumentAnnotationCache.getOrDefault(str1, null); + Annotation jmweAnnotation = jmweAnnotationCache.getOrDefault(str1, null); + if (annotation2 == null) { + createStrAnnotation(str1, stanfordCoreNLPSentiment, true); + } + if (annotation4 == null) { + createStrAnnotation(str1, stanfordCoreNLP, false); + } + if (coreDocument1 == null) { + getCoreDocumentsSuggested(stanfordCoreNLP, str1); + } + if (jmweAnnotation == null) { + getJMWEAnnotation(str1); + jmweAnnotation = jmweAnnotationCache.get(str1); + } + Integer tokenizeCounting = tokenizeCountingHashMap.getOrDefault(str1, null); + + List> taggedWordList1 = taggedWordListHashMap.getOrDefault(str1, null); + + java.util.ArrayList retrieveTGWList1 = retrieveTGWListHashMap.getOrDefault(str1, null); + + List sentence1 = sentences1HashMap.getOrDefault(str1, null); + + List sentenceSentiment1 = sentencesSentimentHashMap.getOrDefault(str1, null); + ArrayList trees1 = trees1HashMap.getOrDefault(str1, null); + List coreMaps2 = new ArrayList<>(); + ArrayList grammaticalStructures1 = grammaticalStructureHashMap.getOrDefault(str1, null); + if (jmweAnnotation != null) { + coreMaps2 = jmweAnnotation.get(CoreAnnotations.SentencesAnnotation.class); + } + ArrayList typedDependencies1 = typedDependenciesHashMap.getOrDefault(str1, null); + ArrayList rnnCoreAnnotationsPredicted1 = rnnCoreAnnotationsPredictedHashMap.getOrDefault(str1, null); + ArrayList simpleMatrices1 = simpleMatricesHashMap.getOrDefault(str1, null); + simpleMatricesHashMap.getOrDefault(str1, null); + ArrayList simpleMatricesNodevectors1 = simpleMatricesNodevectorsHashMap.getOrDefault(str1, null); + List list1 = listHashMap.getOrDefault(str1, null); + Integer longest1 = longestHashMap.getOrDefault(str1, null); + Integer sentimentLongest1 = sentimentHashMap.getOrDefault(str1, null); + List> imwes1 = imwesHashMap.getOrDefault(str1, null); + Integer InflectedCounterNegative1 = InflectedCounterNegativeHashMap.getOrDefault(str1, null); + Integer InflectedCounterPositive1 = InflectedCounterPositiveHashMap.getOrDefault(str1, null); + ArrayList tokenEntry1 = tokenEntryHashMap.getOrDefault(str1, null); + Integer MarkedContinuousCounter1 = MarkedContinuousCounterHashMap.getOrDefault(str1, null); + Integer UnmarkedPatternCounter1 = UnmarkedPatternCounterHashMap.getOrDefault(str1, null); + ArrayList strTokensIpartForm1 = strTokensIpartFormHashMap.getOrDefault(str1, null); + ArrayList tokenForms1 = tokenFormsHashMap.getOrDefault(str1, null); + ArrayList strTokenEntryGetPOS1 = strTokenEntryGetPOSHashMap.getOrDefault(str1, null); + ArrayList intTokenEntyCounts1 = intTokenEntyCountsHashMap.getOrDefault(str1, null); + ArrayList ITokenTags1 = ITokenTagsHashMap.getOrDefault(str1, null); + ArrayList strTokenStems1 = strTokenStemsHashMap.getOrDefault(str1, null); + Integer Anotatorcounter1 = AnotatorcounterHashMap.getOrDefault(str1, null); + Integer TokensCounter1 = TokensCounterHashMap.getOrDefault(str1, null); + ArrayList entityTokenTags1 = entityTokenTagsHashMap.getOrDefault(str1, null); + ArrayList nerEntities1 = nerEntitiesHashMap.getOrDefault(str1, null); + ArrayList nerEntitiesType1 = nerEntitiesTypeHashMap.getOrDefault(str1, null); + ArrayList stopWordToken1 = stopWordTokenHashMap.getOrDefault(str1, null); + ArrayList stopWordLemma1 = stopWordLemmaHashMap.getOrDefault(str1, null); + Integer PairCounter1 = PairCounterHashMap.getOrDefault(str1, null); + + SentimentAnalyzerTest SMX = new SentimentAnalyzerTest(strF, str1, new SimilarityMatrix(strF, str1), + coreMaps1, coreMaps2, strAnno, + pipelineAnnotationCache.get(str1), strAnnoSentiment, + pipelineSentimentAnnotationCache.get(str1), coreDocument, coreDocumentAnnotationCache.get(str1), + tagger, gsf, classifier, tokenizeCounting, tokenizeCountingF, + taggedWordListF, taggedWordList1, retrieveTGWListF, retrieveTGWList1, + sentencesF, sentence1, sentencesSentimentF, sentenceSentiment1, treesF, trees1, + grammaticalStructuresF, grammaticalStructures1, typedDependenciesF, + typedDependencies1, rnnCoreAnnotationsPredictedF, rnnCoreAnnotationsPredicted1, + simpleMatricesF, simpleMatrices1, simpleMatricesNodevectorsF, simpleMatricesNodevectors1, + listF, list1, longestF, longest1, sentimentLongestF, sentimentLongest1, imwesF, + imwes1, InflectedCounterNegativeF, InflectedCounterNegative1, InflectedCounterPositiveF, + InflectedCounterPositive1, tokenEntryF, tokenEntry1, MarkedContinuousCounterF, + MarkedContinuousCounter1, UnmarkedPatternCounterF, UnmarkedPatternCounter1, + strTokensIpartFormF, strTokensIpartForm1, tokenFormsF, tokenForms1, + strTokenEntryGetPOSF, strTokenEntryGetPOS1, intTokenEntyCountsF, + intTokenEntyCounts1, ITokenTagsF, ITokenTags1, strTokenStemsF, strTokenStems1, + AnotatorcounterF, Anotatorcounter1, TokensCounterF, TokensCounter1, + entityTokenTagsF, entityTokenTags1, nerEntitiesF, nerEntities1, nerEntitiesTypeF, + nerEntitiesType1, stopWordTokenF, stopWordToken1, stopWordLemmaF, stopWordLemma1, + PairCounterF, PairCounter1 + ); + if (tokenizeCounting == null) { + tokenizeCountingHashMap.put(str1, SMX.getTokenizeCounting()); + } + if (taggedWordList1 == null) { + taggedWordListHashMap.put(str1, SMX.getTaggedWordList1()); + } + if (tokenizeCountingF == null) { + tokenizeCountingF = SMX.getTokenizeCountingF(); + } + if (taggedWordListF == null) { + taggedWordListF = SMX.getTaggedWordListF(); + } + if (retrieveTGWListF == null) { + retrieveTGWListF = SMX.getRetrieveTGWListF(); + } + if (retrieveTGWList1 == null) { + retrieveTGWListHashMap.put(str1, SMX.getRetrieveTGWList1()); + } + if (sentencesF == null) { + sentencesF = SMX.getSentencesF(); + } + if (sentence1 == null) { + sentences1HashMap.put(str1, SMX.getSentences1()); + } + if (sentencesSentimentF == null) { + sentencesSentimentF = SMX.getSentencesSentimentF(); + } + if (sentenceSentiment1 == null) { + sentencesSentimentHashMap.put(str1, SMX.getSentencesSentiment1()); + } + if (treesF == null) { + treesF = SMX.getTreesF(); + } + if (trees1 == null) { + trees1HashMap.put(str1, SMX.getTrees1()); + } + if (grammaticalStructuresF == null) { + grammaticalStructuresF = SMX.getGrammaticalStructuresF(); + } + if (grammaticalStructures1 == null) { + grammaticalStructureHashMap.put(str1, SMX.getGrammaticalStructures1()); + } + if (typedDependenciesF == null) { + typedDependenciesF = SMX.getTypedDependenciesF(); + } + if (typedDependencies1 == null) { + typedDependenciesHashMap.put(str1, SMX.getTypedDependencies1()); + } + if (rnnCoreAnnotationsPredictedF == null) { + rnnCoreAnnotationsPredictedF = SMX.getRnnCoreAnnotationsPredictedF(); + } + if (rnnCoreAnnotationsPredicted1 == null) { + rnnCoreAnnotationsPredictedHashMap.put(str1, SMX.getRnnCoreAnnotationsPredicted1()); + } + if (simpleMatricesF == null) { + simpleMatricesF = SMX.getSimpleMatricesF(); + } + if (simpleMatrices1 == null) { + simpleMatricesHashMap.put(str1, SMX.getSimpleMatrices1()); + } + if (simpleMatricesNodevectorsF == null) { + simpleMatricesNodevectorsF = SMX.getSimpleMatricesNodevectorsF(); + } + if (simpleMatricesNodevectors1 == null) { + simpleMatricesNodevectorsHashMap.put(str1, SMX.getSimpleMatricesNodevectors1()); + } + if (listF == null) { + listF = SMX.getListF(); + } + if (list1 == null) { + listHashMap.put(str1, SMX.getList1()); + } + if (longestF == null) { + longestF = SMX.getLongestF(); + } + if (longest1 == null) { + longestHashMap.put(str1, SMX.getLongest1()); + } + if (sentimentLongestF == null) { + sentimentLongestF = SMX.getSentimentLongestF(); + } + if (sentimentLongest1 == null) { + sentimentHashMap.put(str1, SMX.getSentimentLongest1()); + } + if (imwesF == null) { + imwesF = SMX.getImwesF(); + } + if (imwes1 == null) { + imwesHashMap.put(str1, SMX.getImwes1()); + } + if (InflectedCounterNegativeF == null) { + InflectedCounterNegativeF = SMX.getInflectedCounterNegativeF(); + } + if (InflectedCounterNegative1 == null) { + InflectedCounterNegativeHashMap.put(str1, SMX.getInflectedCounterNegative1()); + } + if (InflectedCounterPositiveF == null) { + InflectedCounterPositiveF = SMX.getInflectedCounterPositiveF(); + } + if (InflectedCounterPositive1 == null) { + InflectedCounterPositiveHashMap.put(str1, SMX.getInflectedCounterPositive1()); + } + if (tokenEntryF == null) { + tokenEntryF = SMX.getTokenEntryF(); + } + if (tokenEntry1 == null) { + tokenEntryHashMap.put(str1, SMX.getTokenEntry1()); + } + if (MarkedContinuousCounterF == null) { + MarkedContinuousCounterF = SMX.getMarkedContinuousCounterF(); + } + if (MarkedContinuousCounter1 == null) { + MarkedContinuousCounterHashMap.put(str1, SMX.getMarkedContinuousCounter1()); + } + if (UnmarkedPatternCounterF == null) { + UnmarkedPatternCounterF = SMX.getUnmarkedPatternCounterF(); + } + if (UnmarkedPatternCounter1 == null) { + UnmarkedPatternCounterHashMap.put(str1, SMX.getUnmarkedPatternCounter1()); + } + if (strTokensIpartFormF == null) { + strTokensIpartFormF = SMX.getStrTokensIpartFormF(); + } + if (strTokensIpartForm1 == null) { + strTokensIpartFormHashMap.put(str1, SMX.getStrTokensIpartForm1()); + } + if (tokenFormsF == null) { + tokenFormsF = SMX.getTokenFormsF(); + } + if (tokenForms1 == null) { + tokenFormsHashMap.put(str1, SMX.getTokenForms1()); + } + if (strTokenEntryGetPOSF == null) { + strTokenEntryGetPOSF = SMX.getStrTokenEntryGetPOSF(); + } + if (strTokenEntryGetPOS1 == null) { + strTokenEntryGetPOSHashMap.put(str1, SMX.getStrTokenEntryGetPOS1()); + } + if (intTokenEntyCountsF == null) { + intTokenEntyCountsF = SMX.getIntTokenEntyCountsF(); + } + if (intTokenEntyCounts1 == null) { + intTokenEntyCountsHashMap.put(str1, SMX.getIntTokenEntyCounts1()); + } + if (ITokenTagsF == null) { + ITokenTagsF = SMX.getITokenTagsF(); + } + if (ITokenTags1 == null) { + ITokenTagsHashMap.put(str1, SMX.getITokenTags1()); + } + if (strTokenStemsF == null) { + strTokenStemsF = SMX.getStrTokenStemsF(); + } + if (strTokenStems1 == null) { + strTokenStemsHashMap.put(str1, SMX.getStrTokenStems1()); + } + if (AnotatorcounterF == null) { + AnotatorcounterF = SMX.getAnotatorcounterF(); + } + if (Anotatorcounter1 == null) { + AnotatorcounterHashMap.put(str1, SMX.getAnotatorcounter1()); + } + if (TokensCounterF == null) { + TokensCounterF = SMX.getTokensCounterF(); + } + if (TokensCounter1 == null) { + TokensCounterHashMap.put(str1, SMX.getTokensCounter1()); + } + if (entityTokenTagsF == null) { + entityTokenTagsF = SMX.getEntityTokenTagsF(); + } + if (entityTokenTags1 == null) { + entityTokenTagsHashMap.put(str1, SMX.getEntityTokenTags1()); + } + if (nerEntitiesF == null) { + nerEntitiesF = SMX.getNerEntitiesF(); + } + if (nerEntities1 == null) { + nerEntitiesHashMap.put(str1, SMX.getNerEntities1()); + } + if (nerEntitiesTypeF == null) { + nerEntitiesTypeF = SMX.getNerEntitiesTypeF(); + } + if (nerEntitiesType1 == null) { + nerEntitiesTypeHashMap.put(str1, SMX.getNerEntitiesType1()); + } + if (stopWordTokenF == null) { + stopWordTokenF = SMX.getStopWordTokenF(); + } + if (stopWordToken1 == null) { + stopWordTokenHashMap.put(str1, SMX.getStopWordToken1()); + } + if (stopWordLemmaF == null) { + stopWordLemmaF = SMX.getStopWordLemmaF(); + } + if (stopWordLemma1 == null) { + stopWordLemmaHashMap.put(str1, SMX.getStopWordLemma1()); + } + if (PairCounterF == null) { + PairCounterF = SMX.getPairCounterF(); + } + if (PairCounter1 == null) { + PairCounterHashMap.put(str1, SMX.getPairCounter1()); + } + + SimilarityMatrix getSMX = SMX.callSMX(); + double scoreRelationLastUserMsg = getSMX.getDistance(); + if (scoreRelationLastUserMsg > preRelationUserCounters) { + preRelationUserCounters = scoreRelationLastUserMsg; + concurrentRelations.add(getSMX.getSecondaryString()); + } + } + } + int cacheRequirement = 6500; + if (preRelationUserCounters > cacheRequirement && !ues_copy.contains(strF) && filterContent(strF)) { + DataMapper.InsertMYSQLStrings(strF); + DataMapper.checkStringsToDelete(); + } + double randomLenghtPermit = strF.length() * (Math.random() * Math.random() * Math.random() * (Math.random() * 10)); + Collections.reverse(concurrentRelations); + ArrayList mysqlUpdateLastUsed = new ArrayList(); + if (!concurrentRelations.isEmpty()) { + for (String secondaryRelation : concurrentRelations) { + if (SB.toString().length() > randomLenghtPermit && !SB.toString().isEmpty()) { + break; + } + SB.append(secondaryRelation).append(" "); + mysqlUpdateLastUsed.add(secondaryRelation); + } + } + if (SB.toString().isEmpty()) { + return "failure, preventing stuckness"; + } + DataMapper.updateLastUsed(mysqlUpdateLastUsed); + return SB.toString(); + } + + private void getJMWEAnnotation(String str1) { + Annotation jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str1); + jmweAnnotationCache.put(str1, jmweAnnotation); + } + + public String getResponseMsg(String str, String personName, StanfordCoreNLP stanfordCoreNLP, + StanfordCoreNLP stanfordCoreNLPSentiment, Boolean ingameResponse) { + String responseFutures = ""; + String strF = trimString(str); + responseFutures = getResponseFutures(strF, stanfordCoreNLP, stanfordCoreNLPSentiment); + if (!ingameResponse) { + responseFutures = checkPersonPresentInSentence(personName, responseFutures, strF, stanfordCoreNLP, + stanfordCoreNLPSentiment); + } + return responseFutures; + } + + private String checkPersonPresentInSentence(String personName, String responseMsg, String userLastMessage, + StanfordCoreNLP stanfordCoreNLP, StanfordCoreNLP stanfordCoreNLPSentiment) { + try { + CoreDocument pipelineCoreDcoument = new CoreDocument(responseMsg); + CoreDocument pipelineCoreDcoumentLastMsg = new CoreDocument(userLastMessage); + stanfordCoreNLP.annotate(pipelineCoreDcoument); + stanfordCoreNLPSentiment.annotate(pipelineCoreDcoumentLastMsg); + String regex = "(.*?\\d){10,}"; + for (CoreEntityMention em : pipelineCoreDcoument.entityMentions()) { + String entityType = em.entityType(); + if (entityType == "PERSON") { + String str = responseMsg; + String emText = em.text(); + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(personName); + boolean isMatched = matcher.matches(); + if (emText != personName && !isMatched) { + for (CoreEntityMention emLastMsg : pipelineCoreDcoumentLastMsg.entityMentions()) { + if (emText != emLastMsg.text() && !Character.isDigit(Integer.parseInt(emLastMsg.text().trim()))) { + str = (responseMsg.substring(0, responseMsg.indexOf(emText)) + " " + + emLastMsg + " " + responseMsg.substring(responseMsg.indexOf(emText))); + } + } + str += personName; + return str; + } + } + } + } catch (Exception e) { + System.out.println("SCUFFED JAYZ: " + e.getMessage()); + } + return responseMsg; + } + + public boolean filterContent(String str) { + if (!str.isEmpty() && str.length() > 3) { + String str1Local = str.trim(); + if (str1Local.length() > 2 && !str1Local.startsWith("!")) { + return true; + } + } + return false; + } + + public void getCoreDocumentsSuggested(StanfordCoreNLP pipeline, String str) { + Annotation annotation = new Annotation(str); + pipeline.annotate(annotation); + CoreDocument coreDocument = new CoreDocument(annotation); + coreDocumentAnnotationCache.put(str, coreDocument); + } +} diff --git a/ArtificialAutism/src/main/java/FunctionLayer/DatahandlerKotlinObsolete.kt b/ArtificialAutism/src/main/java/FunctionLayer/DatahandlerKotlinObsolete.kt new file mode 100644 index 00000000..7c62f3b1 --- /dev/null +++ b/ArtificialAutism/src/main/java/FunctionLayer/DatahandlerKotlinObsolete.kt @@ -0,0 +1,628 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package FunctionLayer + +import DataLayer.DataMapper +import FunctionLayer.StanfordParser.SentimentAnalyzerTest +import edu.mit.jmwe.data.IMWE +import edu.mit.jmwe.data.IToken +import edu.stanford.nlp.ie.AbstractSequenceClassifier +import edu.stanford.nlp.ie.crf.CRFClassifier +import edu.stanford.nlp.ling.CoreAnnotations +import edu.stanford.nlp.ling.CoreLabel +import edu.stanford.nlp.ling.TaggedWord +import edu.stanford.nlp.parser.lexparser.LexicalizedParser +import edu.stanford.nlp.pipeline.Annotation +import edu.stanford.nlp.pipeline.CoreDocument +import edu.stanford.nlp.pipeline.StanfordCoreNLP +import edu.stanford.nlp.tagger.maxent.MaxentTagger +import edu.stanford.nlp.trees.* +import edu.stanford.nlp.util.CoreMap +import kotlinx.coroutines.* +import org.ejml.simple.SimpleMatrix +import java.util.* +import java.util.concurrent.TimeUnit +import java.util.regex.Pattern +import kotlin.collections.ArrayList +import kotlin.collections.HashMap + + +/** + * + * @author install1 + */ +public class DatahandlerKotlinObsolete { + private var pipelineAnnotationCache: HashMap + private var pipelineSentimentAnnotationCache = HashMap() + private var coreDocumentAnnotationCache: HashMap + private var jmweAnnotationCache = HashMap() + + //private val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz" + private val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz" + private var tagger: MaxentTagger = MaxentTagger() + private var gsf: GrammaticalStructureFactory + private var classifier: AbstractSequenceClassifier + + //SentimentAnalyzer Hashmaps + private var tokenizeCountingHashMap: HashMap = HashMap() + private var taggedWordListHashMap: HashMap>> = HashMap() + private var retrieveTGWListHashMap: HashMap> = + HashMap() + private var sentences1HashMap: HashMap> = HashMap() + private var sentencesSentimentHashMap: HashMap> = HashMap() + private var trees1HashMap: HashMap> = HashMap() + private var grammaticalStructureHashMap: HashMap> = + HashMap() + private var typedDependenciesHashMap: HashMap> = + HashMap() + private var rnnCoreAnnotationsPredictedHashMap: HashMap> = HashMap() + private var simpleMatricesHashMap: HashMap> = HashMap() + private var simpleMatricesNodevectorsHashMap: HashMap> = HashMap() + private var listHashMap: HashMap> = HashMap() + private var longestHashMap: HashMap = HashMap() + private var sentimentHashMap: HashMap = HashMap() + private var imwesHashMap: HashMap>> = HashMap() + private var InflectedCounterNegativeHashMap: HashMap = HashMap() + private var InflectedCounterPositiveHashMap: HashMap = HashMap() + private var tokenEntryHashMap: HashMap> = HashMap() + private var MarkedContinuousCounterHashMap: HashMap = HashMap() + private var UnmarkedPatternCounterHashMap: HashMap = HashMap() + private var strTokensIpartFormHashMap: HashMap> = HashMap() + private var tokenFormsHashMap: HashMap> = HashMap() + private var strTokenEntryGetPOSHashMap: HashMap> = HashMap() + private var intTokenEntyCountsHashMap: HashMap> = HashMap() + private var ITokenTagsHashMap: HashMap> = HashMap() + private var strTokenStemsHashMap: HashMap> = HashMap() + private var AnotatorcounterHashMap: HashMap = HashMap() + private var TokensCounterHashMap: HashMap = HashMap() + private var entityTokenTagsHashMap: HashMap> = HashMap() + private var nerEntitiesHashMap: HashMap> = HashMap() + private var nerEntitiesTypeHashMap: HashMap> = HashMap() + private var stopWordTokenHashMap: HashMap> = HashMap() + private var stopWordLemmaHashMap: HashMap> = HashMap() + private var PairCounterHashMap: HashMap = HashMap() + + constructor() { + jmweAnnotationCache = HashMap() + pipelineAnnotationCache = HashMap() + pipelineSentimentAnnotationCache = HashMap() + coreDocumentAnnotationCache = HashMap() + gsf = initiateGrammaticalStructureFactory() + classifier = CRFClassifier.getClassifierNoExceptions(nerModel) + } + + fun initiateGrammaticalStructureFactory(): GrammaticalStructureFactory { + val options = arrayOf("-maxLength", "100") + //val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz" + val lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz" + val lp = LexicalizedParser.loadModel(lexParserEnglishPCFG, *options) + val tlp = lp.getOp().langpack() + return tlp.grammaticalStructureFactory() + } + + public fun pipeLineSetUp(): StanfordCoreNLP { + val props = Properties() + val shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz" + //val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz" + val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz" + //val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz" + val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.distsim.crf.ser.gz" + props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse") + props.setProperty("parse.model", shiftReduceParserPath) + props.setProperty("parse.maxlen", "90") + props.setProperty("parse.binaryTrees", "true") + props.setProperty("threads", "5") + props.setProperty("pos.maxlen", "90") + props.setProperty("tokenize.maxlen", "90") + props.setProperty("ssplit.maxlen", "90") + props.setProperty("lemma.maxlen", "90") + props.setProperty("ner.model", "$nerModel,$nerModel2,$nerModel3") + props.setProperty("ner.combinationMode", "HIGH_RECALL") + props.setProperty("regexner.ignorecase", "true") + props.setProperty("ner.fine.regexner.ignorecase", "true") + props.setProperty("tokenize.options", "untokenizable=firstKeep") + return StanfordCoreNLP(props) + } + + fun shiftReduceParserInitiate(): StanfordCoreNLP { + val propsSentiment = Properties() + //val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz" + val lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz" + val sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz" + //val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger" + val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words-distsim.tagger" + val customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of," + + "on,or,such,that,the,their,then,there,these,they,this,to,was,will,with" + propsSentiment.setProperty("parse.model", lexParserEnglishPCFG) + propsSentiment.setProperty("sentiment.model", sentimentModel) + propsSentiment.setProperty("parse.maxlen", "90") + propsSentiment.setProperty("threads", "5") + propsSentiment.setProperty("pos.maxlen", "90") + propsSentiment.setProperty("tokenize.maxlen", "90") + propsSentiment.setProperty("ssplit.maxlen", "90") + propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword") //coref too expensive memorywise + propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator") + propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList) + propsSentiment.setProperty("tokenize.options", "untokenizable=firstKeep") + tagger = MaxentTagger(taggerPath) + + println("finished shiftReduceParserInitiate\n") + return StanfordCoreNLP(propsSentiment) + } + + private fun trimString(str: String): String { + var message = str.trim { it <= ' ' } + if (message.startsWith("<@")) { + message = message.substring(message.indexOf("> ") + 2) + } + if (!message.isEmpty()) { + message = message.replace("@", "") + if (message.contains("<>")) { + message = message.substring(message.indexOf(">")) + } + if (message.startsWith("[ *")) { + message = message.substring(message.indexOf("]")) + } + } + return message + } + + private fun createStrAnnotation(str: String, stanfordCoreNLP: StanfordCoreNLP, sentimentBool: Boolean) { + val strAnno2 = Annotation(str) + strAnno2.compact() + stanfordCoreNLP.annotate(strAnno2) + if (sentimentBool) { + pipelineSentimentAnnotationCache.put(str, strAnno2) + } else { + pipelineAnnotationCache.put(str, strAnno2) + } + } + + private fun getResponseFutures(strF: String, stanfordCoreNLP: StanfordCoreNLP, stanfordCoreNLPSentiment: StanfordCoreNLP): String { + val strAnno: Annotation = Annotation(strF) + strAnno.compact() + stanfordCoreNLP.annotate(strAnno) + + val strAnnoSentiment: Annotation = Annotation(strF) + strAnnoSentiment.compact() + stanfordCoreNLPSentiment.annotate(strAnnoSentiment) + + + val annotation = Annotation(strF) + stanfordCoreNLP.annotate(annotation) + val coreDocument = CoreDocument(annotation) + + val values_copy: List = ArrayList(DataMapper.getAllStrings()) + var preRelationUserCounters = -155000.0 + val concurrentRelations: MutableList = arrayListOf() + val SB = StringBuilder() + var jmweAnnotationF = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strF) + var tokenizeCountingF: Int? = null + var taggedWordListF: List>? = null + var retrieveTGWListF: java.util.ArrayList? = null + var sentencesF: List? = null + var sentencesSentimentF: List? = null + var coreMaps1: List = jmweAnnotationF.get(CoreAnnotations.SentencesAnnotation::class.java) + var treesF: java.util.ArrayList? = null + var grammaticalStructuresF: ArrayList? = null + var typedDependenciesF: java.util.ArrayList? = null + var rnnCoreAnnotationsPredictedF: java.util.ArrayList? = null + var simpleMatricesF: java.util.ArrayList? = null + var simpleMatricesNodevectorsF: java.util.ArrayList? = null + var listF: MutableList? = null + var longestF: Int? = null + var sentimentLongestF: Int? = null + var imwesF: List>? = null + var InflectedCounterNegativeF: Int? = null + var InflectedCounterPositiveF: Int? = null + var tokenEntryF: ArrayList? = null + var MarkedContinuousCounterF: Int? = null + var UnmarkedPatternCounterF: Int? = null + var strTokensIpartFormF: ArrayList? = null + var tokenFormsF: java.util.ArrayList? = null + var strTokenEntryGetPOSF: ArrayList? = null + var intTokenEntyCountsF: java.util.ArrayList? = null + var ITokenTagsF: ArrayList? = null + var strTokenStemsF: java.util.ArrayList? = null + var AnotatorcounterF: Int? = null + var TokensCounterF: Int? = null + var entityTokenTagsF: java.util.ArrayList? = null + var nerEntitiesF: java.util.ArrayList? = null + var nerEntitiesTypeF: java.util.ArrayList? = null + var stopWordTokenF: java.util.ArrayList? = null + var stopWordLemmaF: java.util.ArrayList? = null + var PairCounterF: Int? = null + for (str1 in values_copy) { + if (strF != str1) { + val annotation2 = pipelineSentimentAnnotationCache.getOrDefault(str1, null) + val annotation4 = pipelineAnnotationCache.getOrDefault(str1, null) + val coreDocument1 = coreDocumentAnnotationCache.getOrDefault(str1, null) + var jmweAnnotation = jmweAnnotationCache.getOrDefault(str1, null) + if (annotation2 == null) { + createStrAnnotation(str1, stanfordCoreNLPSentiment, true) + } + if (annotation4 == null) { + createStrAnnotation(str1, stanfordCoreNLP, false) + } + if (coreDocument1 == null) { + getCoreDocumentsSuggested(stanfordCoreNLP, str1) + } + if (jmweAnnotation == null) { + getJMWEAnnotation(str1) + jmweAnnotation = jmweAnnotationCache.get(str1) + } + val tokenizeCounting: Int? = tokenizeCountingHashMap.getOrDefault(str1, null) + val taggedWordList1: List>? = taggedWordListHashMap.getOrDefault(str1, null) + val retrieveTGWList1: java.util.ArrayList? = retrieveTGWListHashMap.getOrDefault(str1, null) + val sentence1: List? = sentences1HashMap.getOrDefault(str1, null) + val sentenceSentiment1: List? = sentencesSentimentHashMap.getOrDefault(str1, null) + val trees1 = trees1HashMap.getOrDefault(str1, null) + var coreMaps2: List = listOf() + val grammaticalStructures1 = grammaticalStructureHashMap.getOrDefault( + str1, null) + if (jmweAnnotation != null) { + coreMaps2 = jmweAnnotation.get(CoreAnnotations.SentencesAnnotation::class.java) + } + val typedDependencies1 = typedDependenciesHashMap.getOrDefault(str1, null) + val rnnCoreAnnotationsPredicted1 = rnnCoreAnnotationsPredictedHashMap.getOrDefault(str1, null) + val simpleMatrices1 = simpleMatricesHashMap.getOrDefault(str1, null); + val simpleMatricesNodevectors1 = simpleMatricesNodevectorsHashMap.getOrDefault(str1, null); + val list1 = listHashMap.getOrDefault(str1, null); + val longest1 = longestHashMap.getOrDefault(str1, null); + val sentimentLongest1 = sentimentHashMap.getOrDefault(str1, null); + val imwes1 = imwesHashMap.getOrDefault(str1, null); + val InflectedCounterNegative1 = InflectedCounterNegativeHashMap.getOrDefault(str1, null); + val InflectedCounterPositive1 = InflectedCounterPositiveHashMap.getOrDefault(str1, null) + val tokenEntry1 = tokenEntryHashMap.getOrDefault(str1, null) + val MarkedContinuousCounter1 = MarkedContinuousCounterHashMap.getOrDefault(str1, null) + val UnmarkedPatternCounter1 = UnmarkedPatternCounterHashMap.getOrDefault(str1, null) + val strTokensIpartForm1 = strTokensIpartFormHashMap.getOrDefault(str1, null); + val tokenForms1 = tokenFormsHashMap.getOrDefault(str1, null); + val strTokenEntryGetPOS1 = strTokenEntryGetPOSHashMap.getOrDefault(str1, null) + val intTokenEntyCounts1 = intTokenEntyCountsHashMap.getOrDefault(str1, null); + val ITokenTags1 = ITokenTagsHashMap.getOrDefault(str1, null); + val strTokenStems1 = strTokenStemsHashMap.getOrDefault(str1, null); + val Anotatorcounter1 = AnotatorcounterHashMap.getOrDefault(str1, null); + val TokensCounter1 = TokensCounterHashMap.getOrDefault(str1, null); + val entityTokenTags1 = entityTokenTagsHashMap.getOrDefault(str1, null); + val nerEntities1 = nerEntitiesHashMap.getOrDefault(str1, null); + val nerEntitiesType1 = nerEntitiesTypeHashMap.getOrDefault(str1, null); + val stopWordToken1 = stopWordTokenHashMap.getOrDefault(str1, null); + val stopWordLemma1 = stopWordLemmaHashMap.getOrDefault(str1, null); + val PairCounter1 = PairCounterHashMap.getOrDefault(str1, null); + + var SMX = SentimentAnalyzerTest(strF, str1, SimilarityMatrix(strF, str1), + coreMaps1, coreMaps2, strAnno, + pipelineAnnotationCache[str1], strAnnoSentiment, + pipelineSentimentAnnotationCache[str1], coreDocument, coreDocumentAnnotationCache[str1], + tagger, gsf, classifier, tokenizeCounting, tokenizeCountingF, + taggedWordListF, taggedWordList1, retrieveTGWListF, retrieveTGWList1, + sentencesF, sentence1, sentencesSentimentF, sentenceSentiment1, treesF, trees1, + grammaticalStructuresF, grammaticalStructures1, typedDependenciesF, + typedDependencies1, rnnCoreAnnotationsPredictedF, rnnCoreAnnotationsPredicted1, + simpleMatricesF, simpleMatrices1, simpleMatricesNodevectorsF, simpleMatricesNodevectors1, + listF, list1, longestF, longest1, sentimentLongestF, sentimentLongest1, imwesF, + imwes1, InflectedCounterNegativeF, InflectedCounterNegative1, InflectedCounterPositiveF, + InflectedCounterPositive1, tokenEntryF, tokenEntry1, MarkedContinuousCounterF, + MarkedContinuousCounter1, UnmarkedPatternCounterF, UnmarkedPatternCounter1, + strTokensIpartFormF, strTokensIpartForm1, tokenFormsF, tokenForms1, + strTokenEntryGetPOSF, strTokenEntryGetPOS1, intTokenEntyCountsF, + intTokenEntyCounts1, ITokenTagsF, ITokenTags1, strTokenStemsF, strTokenStems1, + AnotatorcounterF, Anotatorcounter1, TokensCounterF, TokensCounter1, + entityTokenTagsF, entityTokenTags1, nerEntitiesF, nerEntities1, nerEntitiesTypeF, + nerEntitiesType1, stopWordTokenF, stopWordToken1, stopWordLemmaF, stopWordLemma1, + PairCounterF, PairCounter1) + if (tokenizeCounting == null) { + tokenizeCountingHashMap.put(str1, SMX.getTokenizeCounting()) + } + if (taggedWordList1 == null) { + taggedWordListHashMap.put(str1, SMX.getTaggedWordList1()) + } + if (tokenizeCountingF == null) { + tokenizeCountingF = SMX.getTokenizeCountingF(); + } + if (taggedWordListF == null) { + taggedWordListF = SMX.getTaggedWordListF(); + } + if (retrieveTGWListF == null) { + retrieveTGWListF = SMX.getRetrieveTGWListF(); + } + if (retrieveTGWList1 == null) { + retrieveTGWListHashMap.put(str1, SMX.getRetrieveTGWList1()); + } + if (sentencesF == null) { + sentencesF = SMX.getSentencesF(); + } + if (sentence1 == null) { + sentences1HashMap.put(str1, SMX.getSentences1()) + } + if (sentencesSentimentF == null) { + sentencesSentimentF = SMX.getSentencesSentimentF(); + } + if (sentenceSentiment1 == null) { + sentencesSentimentHashMap.put(str1, SMX.getSentencesSentiment1()); + } + if (treesF == null) { + treesF = SMX.getTreesF(); + } + if (trees1 == null) { + trees1HashMap.put(str1, SMX.getTrees1()) + } + if (grammaticalStructuresF == null) { + grammaticalStructuresF = SMX.getGrammaticalStructuresF(); + } + if (grammaticalStructures1 == null) { + grammaticalStructureHashMap.put(str1, SMX.getGrammaticalStructures1()) + } + if (typedDependenciesF == null) { + typedDependenciesF = SMX.getTypedDependenciesF(); + } + if (typedDependencies1 == null) { + typedDependenciesHashMap.put(str1, SMX.getTypedDependencies1()) + } + if (rnnCoreAnnotationsPredictedF == null) { + rnnCoreAnnotationsPredictedF = SMX.getRnnCoreAnnotationsPredictedF() + } + if (rnnCoreAnnotationsPredicted1 == null) { + rnnCoreAnnotationsPredictedHashMap.put(str1, SMX.getRnnCoreAnnotationsPredicted1()) + } + if (simpleMatricesF == null) { + simpleMatricesF = SMX.getSimpleMatricesF(); + } + if (simpleMatrices1 == null) { + simpleMatricesHashMap.put(str1, SMX.getSimpleMatrices1()); + } + if (simpleMatricesNodevectorsF == null) { + simpleMatricesNodevectorsF = SMX.getSimpleMatricesNodevectorsF(); + } + if (simpleMatricesNodevectors1 == null) { + simpleMatricesNodevectorsHashMap.put(str1, SMX.getSimpleMatricesNodevectors1()); + } + if (listF == null) { + listF = SMX.getListF(); + } + if (list1 == null) { + listHashMap.put(str1, SMX.getList1()); + } + if (longestF == null) { + longestF = SMX.getLongestF(); + } + if (longest1 == null) { + longestHashMap.put(str1, SMX.getLongest1()); + } + if (sentimentLongestF == null) { + sentimentLongestF = SMX.getSentimentLongestF(); + } + if (sentimentLongest1 == null) { + sentimentHashMap.put(str1, SMX.getSentimentLongest1()); + } + if (imwesF == null) { + imwesF = SMX.getImwesF(); + } + if (imwes1 == null) { + imwesHashMap.put(str1, SMX.getImwes1()); + } + if (InflectedCounterNegativeF == null) { + InflectedCounterNegativeF = SMX.getInflectedCounterNegativeF(); + } + if (InflectedCounterNegative1 == null) { + InflectedCounterNegativeHashMap.put(str1, SMX.getInflectedCounterNegative1()); + } + if (InflectedCounterPositiveF == null) { + InflectedCounterPositiveF = SMX.getInflectedCounterPositiveF(); + } + if (InflectedCounterPositive1 == null) { + InflectedCounterPositiveHashMap.put(str1, SMX.getInflectedCounterPositive1()); + } + if (tokenEntryF == null) { + tokenEntryF = SMX.getTokenEntryF(); + } + if (tokenEntry1 == null) { + tokenEntryHashMap.put(str1, SMX.getTokenEntry1()) + } + if (MarkedContinuousCounterF == null) { + MarkedContinuousCounterF = SMX.getMarkedContinuousCounterF(); + } + if (MarkedContinuousCounter1 == null) { + MarkedContinuousCounterHashMap.put(str1, SMX.getMarkedContinuousCounter1()); + } + if (UnmarkedPatternCounterF == null) { + UnmarkedPatternCounterF = SMX.getUnmarkedPatternCounterF(); + } + if (UnmarkedPatternCounter1 == null) { + UnmarkedPatternCounterHashMap.put(str1, SMX.getUnmarkedPatternCounter1()); + } + if (strTokensIpartFormF == null) { + strTokensIpartFormF = SMX.getStrTokensIpartFormF(); + } + if (strTokensIpartForm1 == null) { + strTokensIpartFormHashMap.put(str1, SMX.getStrTokensIpartForm1()); + } + if (tokenFormsF == null) { + tokenFormsF = SMX.getTokenFormsF(); + } + if (tokenForms1 == null) { + tokenFormsHashMap.put(str1, SMX.getTokenForms1()); + } + if (strTokenEntryGetPOSF == null) { + strTokenEntryGetPOSF = SMX.getStrTokenEntryGetPOSF(); + } + if (strTokenEntryGetPOS1 == null) { + strTokenEntryGetPOSHashMap.put(str1, SMX.getStrTokenEntryGetPOS1()) + } + if (intTokenEntyCountsF == null) { + intTokenEntyCountsF = SMX.getIntTokenEntyCountsF(); + } + if (intTokenEntyCounts1 == null) { + intTokenEntyCountsHashMap.put(str1, SMX.getIntTokenEntyCounts1()); + } + if (ITokenTagsF == null) { + ITokenTagsF = SMX.getITokenTagsF(); + } + if (ITokenTags1 == null) { + ITokenTagsHashMap.put(str1, SMX.getITokenTags1()); + } + if (strTokenStemsF == null) { + strTokenStemsF = SMX.getStrTokenStemsF(); + } + if (strTokenStems1 == null) { + strTokenStemsHashMap.put(str1, SMX.getStrTokenStems1()); + } + if (AnotatorcounterF == null) { + AnotatorcounterF = SMX.getAnotatorcounterF(); + } + if (Anotatorcounter1 == null) { + AnotatorcounterHashMap.put(str1, SMX.getAnotatorcounter1()); + } + if (TokensCounterF == null) { + TokensCounterF = SMX.getTokensCounterF(); + } + if (TokensCounter1 == null) { + TokensCounterHashMap.put(str1, SMX.getTokensCounter1()); + } + if (entityTokenTagsF == null) { + entityTokenTagsF = SMX.getEntityTokenTagsF(); + } + if (entityTokenTags1 == null) { + entityTokenTagsHashMap.put(str1, SMX.getEntityTokenTags1()); + } + if (nerEntitiesF == null) { + nerEntitiesF = SMX.getNerEntitiesF(); + } + if (nerEntities1 == null) { + nerEntitiesHashMap.put(str1, SMX.getNerEntities1()); + } + if (nerEntitiesTypeF == null) { + nerEntitiesTypeF = SMX.getNerEntitiesTypeF(); + } + if (nerEntitiesType1 == null) { + nerEntitiesTypeHashMap.put(str1, SMX.getNerEntitiesType1()); + } + if (stopWordTokenF == null) { + stopWordTokenF = SMX.getStopWordTokenF(); + } + if (stopWordToken1 == null) { + stopWordTokenHashMap.put(str1, SMX.getStopWordToken1()); + } + if (stopWordLemmaF == null) { + stopWordLemmaF = SMX.getStopWordLemmaF(); + } + if (stopWordLemma1 == null) { + stopWordLemmaHashMap.put(str1, SMX.getStopWordLemma1()); + } + if (PairCounterF == null) { + PairCounterF = SMX.getPairCounterF(); + } + if (PairCounter1 == null) { + PairCounterHashMap.put(str1, SMX.getPairCounter1()); + } + + var getSMX: SimilarityMatrix = SMX.callSMX() + val scoreRelationLastUserMsg = getSMX.distance + if (scoreRelationLastUserMsg > preRelationUserCounters) { + preRelationUserCounters = scoreRelationLastUserMsg + concurrentRelations.add(getSMX.secondaryString) + } + } + } + val cacheRequirement = 6500; + if (preRelationUserCounters > cacheRequirement && !values_copy.contains(strF) && filterContent(strF)) { + DataMapper.InsertMYSQLStrings(strF) + DataMapper.checkStringsToDelete(); + } + val randomLenghtPermit = strF.length * (Math.random() * Math.random() * Math.random() * (Math.random() * 10)) + Collections.reverse(concurrentRelations) + val mysqlUpdateLastUsed: ArrayList = ArrayList() + if (!concurrentRelations.isEmpty()) { + for (secondaryRelation in concurrentRelations) { + if (SB.toString().length > randomLenghtPermit && !SB.toString().isEmpty()) { + break + } + SB.append(secondaryRelation).append(" ") + mysqlUpdateLastUsed.add(secondaryRelation) + } + } + if (SB.toString().isEmpty()) { + return "failure, preventing stuckness" + } + DataMapper.updateLastUsed(mysqlUpdateLastUsed); + return SB.toString() + } + + private fun getJMWEAnnotation(str1: String) { + val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str1) + jmweAnnotationCache.put(str1, jmweAnnotation) + } + + fun getResponseMsg(str: String, personName: String, stanfordCoreNLP: StanfordCoreNLP, + stanfordCoreNLPSentiment: StanfordCoreNLP, ingameResponse: Boolean): String { + var responseFutures: String = "" + runBlocking { + launch(Dispatchers.Default) { + var strF = trimString(str) + responseFutures = getResponseFutures(strF, stanfordCoreNLP, stanfordCoreNLPSentiment) + if (!ingameResponse) { + responseFutures = checkPersonPresentInSentence(personName, responseFutures, strF, stanfordCoreNLP, + stanfordCoreNLPSentiment) + } + yield() + }.join() + } + return responseFutures + } + + private fun checkPersonPresentInSentence(personName: String, responseMsg: String, userLastMessage: String, + stanfordCoreNLP: StanfordCoreNLP, + stanfordCoreNLPSentiment: StanfordCoreNLP): String { + try { + val pipelineCoreDcoument = CoreDocument(responseMsg) + val pipelineCoreDcoumentLastMsg = CoreDocument(userLastMessage) + stanfordCoreNLP.annotate(pipelineCoreDcoument) + stanfordCoreNLPSentiment.annotate(pipelineCoreDcoumentLastMsg) + val regex = "(.*?\\d){10,}" + for (em in pipelineCoreDcoument.entityMentions()) { + val entityType = em.entityType() + if (entityType == "PERSON") { + var str = responseMsg + val emText = em.text() + val pattern = Pattern.compile(regex) + val matcher = pattern.matcher(personName) + val isMatched = matcher.matches() + if (emText != personName && !isMatched) { + for (emLastMsg in pipelineCoreDcoumentLastMsg.entityMentions()) { + if (emText != emLastMsg.text() && !Character.isDigit(emLastMsg.text().trim { it <= ' ' }[0])) { + str = (responseMsg.substring(0, responseMsg.indexOf(emText)) + " " + + emLastMsg + " " + responseMsg.substring(responseMsg.indexOf(emText))) + } + } + str += " $personName" + return str + } + } + } + } catch (e: Exception) { + println("""SCUFFED JAYZ: ${e.message}""".trimIndent()) + } + return responseMsg + } + + fun filterContent(str: String): Boolean { + if (!str.isEmpty() && str.length > 3) { + var str1Local: String = str.trim(); + if (str1Local.length > 2 && !str1Local.startsWith("!")) { + return true + } + } + return false + } + + fun getCoreDocumentsSuggested(pipeline: StanfordCoreNLP, str: String) { + val annotation = Annotation(str) + pipeline.annotate(annotation) + val coreDocument = CoreDocument(annotation) + coreDocumentAnnotationCache.put(str, coreDocument) + } +} \ No newline at end of file diff --git a/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java b/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java index 5d3ea85f..efeca5de 100644 --- a/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java +++ b/ArtificialAutism/src/main/java/FunctionLayer/DoStuff.java @@ -5,20 +5,13 @@ */ package FunctionLayer; -import PresentationLayer.DiscordHandler; import discord4j.core.event.domain.message.MessageCreateEvent; import discord4j.core.object.entity.User; import discord4j.core.object.entity.channel.TextChannel; -import java.math.BigInteger; -import java.util.ArrayList; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; import edu.stanford.nlp.pipeline.StanfordCoreNLP; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; /** diff --git a/ArtificialAutism/src/test/java/junit.java b/ArtificialAutism/src/test/java/junit.java index a62c8a9c..f5d0e35f 100644 --- a/ArtificialAutism/src/test/java/junit.java +++ b/ArtificialAutism/src/test/java/junit.java @@ -1,4 +1,4 @@ -import FunctionLayer.Datahandler; +import FunctionLayer.DatahandlerKotlinObsolete; import FunctionLayer.PipelineJMWESingleton; import FunctionLayer.StanfordParser.SentimentAnalyzerTest; import edu.mit.jmwe.data.IMWE; @@ -17,10 +17,9 @@ import edu.stanford.nlp.trees.*; import edu.stanford.nlp.util.CoreMap; import org.ejml.simple.SimpleMatrix; import org.junit.Assert; -import org.junit.Test; import FunctionLayer.SimilarityMatrix; +import org.junit.Test; -import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -178,7 +177,7 @@ public class junit { //@Test public void testScoring() { - Datahandler datahandler = new Datahandler(); + DatahandlerKotlinObsolete datahandler = new DatahandlerKotlinObsolete(); PipelineJMWESingleton.getINSTANCE(); StanfordCoreNLP stanfordCoreNLP = datahandler.pipeLineSetUp(); StanfordCoreNLP stanfordCoreNLPSentiment = datahandler.shiftReduceParserInitiate();