/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package FunctionLayer

import DataLayer.DataMapper
import FunctionLayer.StanfordParser.SentimentAnalyzerTest
import edu.mit.jmwe.data.IMWE
import edu.mit.jmwe.data.IToken
import edu.stanford.nlp.ie.AbstractSequenceClassifier
import edu.stanford.nlp.ie.crf.CRFClassifier
import edu.stanford.nlp.ling.CoreAnnotations
import edu.stanford.nlp.ling.CoreLabel
import edu.stanford.nlp.ling.TaggedWord
import edu.stanford.nlp.parser.lexparser.LexicalizedParser
import edu.stanford.nlp.pipeline.Annotation
import edu.stanford.nlp.pipeline.CoreDocument
import edu.stanford.nlp.pipeline.StanfordCoreNLP
import edu.stanford.nlp.tagger.maxent.MaxentTagger
import edu.stanford.nlp.trees.*
import edu.stanford.nlp.util.CoreMap
import kotlinx.coroutines.*
import org.ejml.simple.SimpleMatrix
import java.util.*
import java.util.concurrent.TimeUnit
import java.util.regex.Pattern
import kotlin.collections.ArrayList
import kotlin.collections.HashMap


/**
 *
 * @author install1
 */
public class Datahandler {
    private var pipelineAnnotationCache: HashMap<String, Annotation>
    private var pipelineSentimentAnnotationCache = HashMap<String, Annotation>()
    private var coreDocumentAnnotationCache: HashMap<String, CoreDocument>
    private var jmweAnnotationCache = HashMap<String, Annotation>()

    //private val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz"
    private val nerModel = "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz"
    private var tagger: MaxentTagger = MaxentTagger()
    private var gsf: GrammaticalStructureFactory
    private var classifier: AbstractSequenceClassifier<CoreLabel>

    //SentimentAnalyzer Hashmaps
    private var tokenizeCountingHashMap: HashMap<String, Int> = HashMap()
    private var taggedWordListHashMap: HashMap<String, List<List<TaggedWord>>> = HashMap()
    private var retrieveTGWListHashMap: HashMap<String, java.util.ArrayList<String>> =
            HashMap()
    private var sentences1HashMap: HashMap<String, List<CoreMap>> = HashMap()
    private var sentencesSentimentHashMap: HashMap<String, List<CoreMap>> = HashMap()
    private var trees1HashMap: HashMap<String, java.util.ArrayList<Tree>> = HashMap()
    private var grammaticalStructureHashMap: HashMap<String, java.util.ArrayList<GrammaticalStructure>> =
            HashMap()
    private var typedDependenciesHashMap: HashMap<String, java.util.ArrayList<TypedDependency>> =
            HashMap()
    private var rnnCoreAnnotationsPredictedHashMap: HashMap<String, java.util.ArrayList<Int>> = HashMap()
    private var simpleMatricesHashMap: HashMap<String, java.util.ArrayList<SimpleMatrix>> = HashMap()
    private var simpleMatricesNodevectorsHashMap: HashMap<String, java.util.ArrayList<SimpleMatrix>> = HashMap()
    private var listHashMap: HashMap<String, MutableList<Any?>> = HashMap()
    private var longestHashMap: HashMap<String, Int> = HashMap()
    private var sentimentHashMap: HashMap<String, Int> = HashMap()
    private var imwesHashMap: HashMap<String, List<IMWE<IToken>>> = HashMap()
    private var InflectedCounterNegativeHashMap: HashMap<String, Int> = HashMap()
    private var InflectedCounterPositiveHashMap: HashMap<String, Int> = HashMap()
    private var tokenEntryHashMap: HashMap<String, ArrayList<String>> = HashMap()
    private var MarkedContinuousCounterHashMap: HashMap<String, Int> = HashMap()
    private var UnmarkedPatternCounterHashMap: HashMap<String, Int> = HashMap()
    private var strTokensIpartFormHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
    private var tokenFormsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
    private var strTokenEntryGetPOSHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
    private var intTokenEntyCountsHashMap: HashMap<String, java.util.ArrayList<Int>> = HashMap()
    private var ITokenTagsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
    private var strTokenStemsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
    private var AnotatorcounterHashMap: HashMap<String, Int> = HashMap()
    private var TokensCounterHashMap: HashMap<String, Int> = HashMap()
    private var entityTokenTagsHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
    private var nerEntitiesHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
    private var nerEntitiesTypeHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
    private var stopWordTokenHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
    private var stopWordLemmaHashMap: HashMap<String, java.util.ArrayList<String>> = HashMap()
    private var PairCounterHashMap: HashMap<String, Int> = HashMap()

    constructor() {
        jmweAnnotationCache = HashMap<String, Annotation>()
        pipelineAnnotationCache = HashMap<String, Annotation>()
        pipelineSentimentAnnotationCache = HashMap<String, Annotation>()
        coreDocumentAnnotationCache = HashMap<String, CoreDocument>()
        gsf = initiateGrammaticalStructureFactory()
        classifier = CRFClassifier.getClassifierNoExceptions(nerModel)
    }

    fun initiateGrammaticalStructureFactory(): GrammaticalStructureFactory {
        val options = arrayOf("-maxLength", "100")
        //val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
        val lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"
        val lp = LexicalizedParser.loadModel(lexParserEnglishPCFG, *options)
        val tlp = lp.getOp().langpack()
        return tlp.grammaticalStructureFactory()
    }

    public fun pipeLineSetUp(): StanfordCoreNLP {
        val props = Properties()
        val shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz"
        //val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz"
        val nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz"
        //val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz"
        val nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.distsim.crf.ser.gz"
        props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse")
        props.setProperty("parse.model", shiftReduceParserPath)
        props.setProperty("parse.maxlen", "90")
        props.setProperty("parse.binaryTrees", "true")
        props.setProperty("threads", "5")
        props.setProperty("pos.maxlen", "90")
        props.setProperty("tokenize.maxlen", "90")
        props.setProperty("ssplit.maxlen", "90")
        props.setProperty("lemma.maxlen", "90")
        props.setProperty("ner.model", "$nerModel,$nerModel2,$nerModel3")
        props.setProperty("ner.combinationMode", "HIGH_RECALL")
        props.setProperty("regexner.ignorecase", "true")
        props.setProperty("ner.fine.regexner.ignorecase", "true")
        props.setProperty("tokenize.options", "untokenizable=firstKeep")
        return StanfordCoreNLP(props)
    }

    fun shiftReduceParserInitiate(): StanfordCoreNLP {
        val propsSentiment = Properties()
        //val lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz"
        val lexParserEnglishPCFG = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"
        val sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz"
        //val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger"
        val taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words-distsim.tagger"
        val customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of," +
                "on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"
        propsSentiment.setProperty("parse.model", lexParserEnglishPCFG)
        propsSentiment.setProperty("sentiment.model", sentimentModel)
        propsSentiment.setProperty("parse.maxlen", "90")
        propsSentiment.setProperty("threads", "5")
        propsSentiment.setProperty("pos.maxlen", "90")
        propsSentiment.setProperty("tokenize.maxlen", "90")
        propsSentiment.setProperty("ssplit.maxlen", "90")
        propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword") //coref too expensive memorywise
        propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator")
        propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList)
        propsSentiment.setProperty("tokenize.options", "untokenizable=firstKeep")
        tagger = MaxentTagger(taggerPath)

        println("finished shiftReduceParserInitiate\n")
        return StanfordCoreNLP(propsSentiment)
    }

    private fun trimString(str: String): String {
        var message = str.trim { it <= ' ' }
        if (message.startsWith("<@")) {
            message = message.substring(message.indexOf("> ") + 2)
        }
        if (!message.isEmpty()) {
            message = message.replace("@", "")
            if (message.contains("<>")) {
                message = message.substring(message.indexOf(">"))
            }
            if (message.startsWith("[ *")) {
                message = message.substring(message.indexOf("]"))
            }
        }
        return message
    }

    private fun createStrAnnotation(str: String, stanfordCoreNLP: StanfordCoreNLP, sentimentBool: Boolean) {
        val strAnno2 = Annotation(str)
        strAnno2.compact()
        stanfordCoreNLP.annotate(strAnno2)
        if (sentimentBool) {
            pipelineSentimentAnnotationCache.put(str, strAnno2)
        } else {
            pipelineAnnotationCache.put(str, strAnno2)
        }
    }

    private fun getResponseFutures(strF: String, stanfordCoreNLP: StanfordCoreNLP, stanfordCoreNLPSentiment: StanfordCoreNLP): String {
        val strAnno: Annotation = Annotation(strF)
        strAnno.compact()
        stanfordCoreNLP.annotate(strAnno)

        val strAnnoSentiment: Annotation = Annotation(strF)
        strAnnoSentiment.compact()
        stanfordCoreNLPSentiment.annotate(strAnnoSentiment)


        val annotation = Annotation(strF)
        stanfordCoreNLP.annotate(annotation)
        val coreDocument = CoreDocument(annotation)

        val values_copy: List<String> = ArrayList(DataMapper.getAllStrings())
        var preRelationUserCounters = -155000.0
        val concurrentRelations: MutableList<String> = arrayListOf()
        val SB = StringBuilder()
        var jmweAnnotationF = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strF)
        var tokenizeCountingF: Int? = null
        var taggedWordListF: List<List<TaggedWord>>? = null
        var retrieveTGWListF: java.util.ArrayList<String>? = null
        var sentencesF: List<CoreMap>? = null
        var sentencesSentimentF: List<CoreMap>? = null
        var coreMaps1: List<CoreMap> = jmweAnnotationF.get(CoreAnnotations.SentencesAnnotation::class.java)
        var treesF: java.util.ArrayList<Tree>? = null
        var grammaticalStructuresF: ArrayList<GrammaticalStructure>? = null
        var typedDependenciesF: java.util.ArrayList<TypedDependency>? = null
        var rnnCoreAnnotationsPredictedF: java.util.ArrayList<Int>? = null
        var simpleMatricesF: java.util.ArrayList<SimpleMatrix>? = null
        var simpleMatricesNodevectorsF: java.util.ArrayList<SimpleMatrix>? = null
        var listF: MutableList<Any?>? = null
        var longestF: Int? = null
        var sentimentLongestF: Int? = null
        var imwesF: List<IMWE<IToken>>? = null
        var InflectedCounterNegativeF: Int? = null
        var InflectedCounterPositiveF: Int? = null
        var tokenEntryF: ArrayList<String>? = null
        var MarkedContinuousCounterF: Int? = null
        var UnmarkedPatternCounterF: Int? = null
        var strTokensIpartFormF: ArrayList<String>? = null
        var tokenFormsF: java.util.ArrayList<String>? = null
        var strTokenEntryGetPOSF: ArrayList<String>? = null
        var intTokenEntyCountsF: java.util.ArrayList<Int>? = null
        var ITokenTagsF: ArrayList<String>? = null
        var strTokenStemsF: java.util.ArrayList<String>? = null
        var AnotatorcounterF: Int? = null
        var TokensCounterF: Int? = null
        var entityTokenTagsF: java.util.ArrayList<String>? = null
        var nerEntitiesF: java.util.ArrayList<String>? = null
        var nerEntitiesTypeF: java.util.ArrayList<String>? = null
        var stopWordTokenF: java.util.ArrayList<String>? = null
        var stopWordLemmaF: java.util.ArrayList<String>? = null
        var PairCounterF: Int? = null
        for (str1 in values_copy) {
            if (strF != str1) {
                val annotation2 = pipelineSentimentAnnotationCache.getOrDefault(str1, null)
                val annotation4 = pipelineAnnotationCache.getOrDefault(str1, null)
                val coreDocument1 = coreDocumentAnnotationCache.getOrDefault(str1, null)
                var jmweAnnotation = jmweAnnotationCache.getOrDefault(str1, null)
                if (annotation2 == null) {
                    createStrAnnotation(str1, stanfordCoreNLPSentiment, true)
                }
                if (annotation4 == null) {
                    createStrAnnotation(str1, stanfordCoreNLP, false)
                }
                if (coreDocument1 == null) {
                    getCoreDocumentsSuggested(stanfordCoreNLP, str1)
                }
                if (jmweAnnotation == null) {
                    getJMWEAnnotation(str1)
                    jmweAnnotation = jmweAnnotationCache.get(str1)
                }
                val tokenizeCounting: Int? = tokenizeCountingHashMap.getOrDefault(str1, null)
                val taggedWordList1: List<List<TaggedWord>>? = taggedWordListHashMap.getOrDefault(str1, null)
                val retrieveTGWList1: java.util.ArrayList<String>? = retrieveTGWListHashMap.getOrDefault(str1, null)
                val sentence1: List<CoreMap>? = sentences1HashMap.getOrDefault(str1, null)
                val sentenceSentiment1: List<CoreMap>? = sentencesSentimentHashMap.getOrDefault(str1, null)
                val trees1 = trees1HashMap.getOrDefault(str1, null)
                var coreMaps2: List<CoreMap> = listOf()
                val grammaticalStructures1 = grammaticalStructureHashMap.getOrDefault(
                        str1, null)
                if (jmweAnnotation != null) {
                    coreMaps2 = jmweAnnotation.get(CoreAnnotations.SentencesAnnotation::class.java)
                }
                val typedDependencies1 = typedDependenciesHashMap.getOrDefault(str1, null)
                val rnnCoreAnnotationsPredicted1 = rnnCoreAnnotationsPredictedHashMap.getOrDefault(str1, null)
                val simpleMatrices1 = simpleMatricesHashMap.getOrDefault(str1, null);
                val simpleMatricesNodevectors1 = simpleMatricesNodevectorsHashMap.getOrDefault(str1, null);
                val list1 = listHashMap.getOrDefault(str1, null);
                val longest1 = longestHashMap.getOrDefault(str1, null);
                val sentimentLongest1 = sentimentHashMap.getOrDefault(str1, null);
                val imwes1 = imwesHashMap.getOrDefault(str1, null);
                val InflectedCounterNegative1 = InflectedCounterNegativeHashMap.getOrDefault(str1, null);
                val InflectedCounterPositive1 = InflectedCounterPositiveHashMap.getOrDefault(str1, null)
                val tokenEntry1 = tokenEntryHashMap.getOrDefault(str1, null)
                val MarkedContinuousCounter1 = MarkedContinuousCounterHashMap.getOrDefault(str1, null)
                val UnmarkedPatternCounter1 = UnmarkedPatternCounterHashMap.getOrDefault(str1, null)
                val strTokensIpartForm1 = strTokensIpartFormHashMap.getOrDefault(str1, null);
                val tokenForms1 = tokenFormsHashMap.getOrDefault(str1, null);
                val strTokenEntryGetPOS1 = strTokenEntryGetPOSHashMap.getOrDefault(str1, null)
                val intTokenEntyCounts1 = intTokenEntyCountsHashMap.getOrDefault(str1, null);
                val ITokenTags1 = ITokenTagsHashMap.getOrDefault(str1, null);
                val strTokenStems1 = strTokenStemsHashMap.getOrDefault(str1, null);
                val Anotatorcounter1 = AnotatorcounterHashMap.getOrDefault(str1, null);
                val TokensCounter1 = TokensCounterHashMap.getOrDefault(str1, null);
                val entityTokenTags1 = entityTokenTagsHashMap.getOrDefault(str1, null);
                val nerEntities1 = nerEntitiesHashMap.getOrDefault(str1, null);
                val nerEntitiesType1 = nerEntitiesTypeHashMap.getOrDefault(str1, null);
                val stopWordToken1 = stopWordTokenHashMap.getOrDefault(str1, null);
                val stopWordLemma1 = stopWordLemmaHashMap.getOrDefault(str1, null);
                val PairCounter1 = PairCounterHashMap.getOrDefault(str1, null);

                var SMX = SentimentAnalyzerTest(strF, str1, SimilarityMatrix(strF, str1),
                        coreMaps1, coreMaps2, strAnno,
                        pipelineAnnotationCache[str1], strAnnoSentiment,
                        pipelineSentimentAnnotationCache[str1], coreDocument, coreDocumentAnnotationCache[str1],
                        tagger, gsf, classifier, tokenizeCounting, tokenizeCountingF,
                        taggedWordListF, taggedWordList1, retrieveTGWListF, retrieveTGWList1,
                        sentencesF, sentence1, sentencesSentimentF, sentenceSentiment1, treesF, trees1,
                        grammaticalStructuresF, grammaticalStructures1, typedDependenciesF,
                        typedDependencies1, rnnCoreAnnotationsPredictedF, rnnCoreAnnotationsPredicted1,
                        simpleMatricesF, simpleMatrices1, simpleMatricesNodevectorsF, simpleMatricesNodevectors1,
                        listF, list1, longestF, longest1, sentimentLongestF, sentimentLongest1, imwesF,
                        imwes1, InflectedCounterNegativeF, InflectedCounterNegative1, InflectedCounterPositiveF,
                        InflectedCounterPositive1, tokenEntryF, tokenEntry1, MarkedContinuousCounterF,
                        MarkedContinuousCounter1, UnmarkedPatternCounterF, UnmarkedPatternCounter1,
                        strTokensIpartFormF, strTokensIpartForm1, tokenFormsF, tokenForms1,
                        strTokenEntryGetPOSF, strTokenEntryGetPOS1, intTokenEntyCountsF,
                        intTokenEntyCounts1, ITokenTagsF, ITokenTags1, strTokenStemsF, strTokenStems1,
                        AnotatorcounterF, Anotatorcounter1, TokensCounterF, TokensCounter1,
                        entityTokenTagsF, entityTokenTags1, nerEntitiesF, nerEntities1, nerEntitiesTypeF,
                        nerEntitiesType1, stopWordTokenF, stopWordToken1, stopWordLemmaF, stopWordLemma1,
                        PairCounterF, PairCounter1)
                if (tokenizeCounting == null) {
                    tokenizeCountingHashMap.put(str1, SMX.getTokenizeCounting())
                }
                if (taggedWordList1 == null) {
                    taggedWordListHashMap.put(str1, SMX.getTaggedWordList1())
                }
                if (tokenizeCountingF == null) {
                    tokenizeCountingF = SMX.getTokenizeCountingF();
                }
                if (taggedWordListF == null) {
                    taggedWordListF = SMX.getTaggedWordListF();
                }
                if (retrieveTGWListF == null) {
                    retrieveTGWListF = SMX.getRetrieveTGWListF();
                }
                if (retrieveTGWList1 == null) {
                    retrieveTGWListHashMap.put(str1, SMX.getRetrieveTGWList1());
                }
                if (sentencesF == null) {
                    sentencesF = SMX.getSentencesF();
                }
                if (sentence1 == null) {
                    sentences1HashMap.put(str1, SMX.getSentences1())
                }
                if (sentencesSentimentF == null) {
                    sentencesSentimentF = SMX.getSentencesSentimentF();
                }
                if (sentenceSentiment1 == null) {
                    sentencesSentimentHashMap.put(str1, SMX.getSentencesSentiment1());
                }
                if (treesF == null) {
                    treesF = SMX.getTreesF();
                }
                if (trees1 == null) {
                    trees1HashMap.put(str1, SMX.getTrees1())
                }
                if (grammaticalStructuresF == null) {
                    grammaticalStructuresF = SMX.getGrammaticalStructuresF();
                }
                if (grammaticalStructures1 == null) {
                    grammaticalStructureHashMap.put(str1, SMX.getGrammaticalStructures1())
                }
                if (typedDependenciesF == null) {
                    typedDependenciesF = SMX.getTypedDependenciesF();
                }
                if (typedDependencies1 == null) {
                    typedDependenciesHashMap.put(str1, SMX.getTypedDependencies1())
                }
                if (rnnCoreAnnotationsPredictedF == null) {
                    rnnCoreAnnotationsPredictedF = SMX.getRnnCoreAnnotationsPredictedF()
                }
                if (rnnCoreAnnotationsPredicted1 == null) {
                    rnnCoreAnnotationsPredictedHashMap.put(str1, SMX.getRnnCoreAnnotationsPredicted1())
                }
                if (simpleMatricesF == null) {
                    simpleMatricesF = SMX.getSimpleMatricesF();
                }
                if (simpleMatrices1 == null) {
                    simpleMatricesHashMap.put(str1, SMX.getSimpleMatrices1());
                }
                if (simpleMatricesNodevectorsF == null) {
                    simpleMatricesNodevectorsF = SMX.getSimpleMatricesNodevectorsF();
                }
                if (simpleMatricesNodevectors1 == null) {
                    simpleMatricesNodevectorsHashMap.put(str1, SMX.getSimpleMatricesNodevectors1());
                }
                if (listF == null) {
                    listF = SMX.getListF();
                }
                if (list1 == null) {
                    listHashMap.put(str1, SMX.getList1());
                }
                if (longestF == null) {
                    longestF = SMX.getLongestF();
                }
                if (longest1 == null) {
                    longestHashMap.put(str1, SMX.getLongest1());
                }
                if (sentimentLongestF == null) {
                    sentimentLongestF = SMX.getSentimentLongestF();
                }
                if (sentimentLongest1 == null) {
                    sentimentHashMap.put(str1, SMX.getSentimentLongest1());
                }
                if (imwesF == null) {
                    imwesF = SMX.getImwesF();
                }
                if (imwes1 == null) {
                    imwesHashMap.put(str1, SMX.getImwes1());
                }
                if (InflectedCounterNegativeF == null) {
                    InflectedCounterNegativeF = SMX.getInflectedCounterNegativeF();
                }
                if (InflectedCounterNegative1 == null) {
                    InflectedCounterNegativeHashMap.put(str1, SMX.getInflectedCounterNegative1());
                }
                if (InflectedCounterPositiveF == null) {
                    InflectedCounterPositiveF = SMX.getInflectedCounterPositiveF();
                }
                if (InflectedCounterPositive1 == null) {
                    InflectedCounterPositiveHashMap.put(str1, SMX.getInflectedCounterPositive1());
                }
                if (tokenEntryF == null) {
                    tokenEntryF = SMX.getTokenEntryF();
                }
                if (tokenEntry1 == null) {
                    tokenEntryHashMap.put(str1, SMX.getTokenEntry1())
                }
                if (MarkedContinuousCounterF == null) {
                    MarkedContinuousCounterF = SMX.getMarkedContinuousCounterF();
                }
                if (MarkedContinuousCounter1 == null) {
                    MarkedContinuousCounterHashMap.put(str1, SMX.getMarkedContinuousCounter1());
                }
                if (UnmarkedPatternCounterF == null) {
                    UnmarkedPatternCounterF = SMX.getUnmarkedPatternCounterF();
                }
                if (UnmarkedPatternCounter1 == null) {
                    UnmarkedPatternCounterHashMap.put(str1, SMX.getUnmarkedPatternCounter1());
                }
                if (strTokensIpartFormF == null) {
                    strTokensIpartFormF = SMX.getStrTokensIpartFormF();
                }
                if (strTokensIpartForm1 == null) {
                    strTokensIpartFormHashMap.put(str1, SMX.getStrTokensIpartForm1());
                }
                if (tokenFormsF == null) {
                    tokenFormsF = SMX.getTokenFormsF();
                }
                if (tokenForms1 == null) {
                    tokenFormsHashMap.put(str1, SMX.getTokenForms1());
                }
                if (strTokenEntryGetPOSF == null) {
                    strTokenEntryGetPOSF = SMX.getStrTokenEntryGetPOSF();
                }
                if (strTokenEntryGetPOS1 == null) {
                    strTokenEntryGetPOSHashMap.put(str1, SMX.getStrTokenEntryGetPOS1())
                }
                if (intTokenEntyCountsF == null) {
                    intTokenEntyCountsF = SMX.getIntTokenEntyCountsF();
                }
                if (intTokenEntyCounts1 == null) {
                    intTokenEntyCountsHashMap.put(str1, SMX.getIntTokenEntyCounts1());
                }
                if (ITokenTagsF == null) {
                    ITokenTagsF = SMX.getITokenTagsF();
                }
                if (ITokenTags1 == null) {
                    ITokenTagsHashMap.put(str1, SMX.getITokenTags1());
                }
                if (strTokenStemsF == null) {
                    strTokenStemsF = SMX.getStrTokenStemsF();
                }
                if (strTokenStems1 == null) {
                    strTokenStemsHashMap.put(str1, SMX.getStrTokenStems1());
                }
                if (AnotatorcounterF == null) {
                    AnotatorcounterF = SMX.getAnotatorcounterF();
                }
                if (Anotatorcounter1 == null) {
                    AnotatorcounterHashMap.put(str1, SMX.getAnotatorcounter1());
                }
                if (TokensCounterF == null) {
                    TokensCounterF = SMX.getTokensCounterF();
                }
                if (TokensCounter1 == null) {
                    TokensCounterHashMap.put(str1, SMX.getTokensCounter1());
                }
                if (entityTokenTagsF == null) {
                    entityTokenTagsF = SMX.getEntityTokenTagsF();
                }
                if (entityTokenTags1 == null) {
                    entityTokenTagsHashMap.put(str1, SMX.getEntityTokenTags1());
                }
                if (nerEntitiesF == null) {
                    nerEntitiesF = SMX.getNerEntitiesF();
                }
                if (nerEntities1 == null) {
                    nerEntitiesHashMap.put(str1, SMX.getNerEntities1());
                }
                if (nerEntitiesTypeF == null) {
                    nerEntitiesTypeF = SMX.getNerEntitiesTypeF();
                }
                if (nerEntitiesType1 == null) {
                    nerEntitiesTypeHashMap.put(str1, SMX.getNerEntitiesType1());
                }
                if (stopWordTokenF == null) {
                    stopWordTokenF = SMX.getStopWordTokenF();
                }
                if (stopWordToken1 == null) {
                    stopWordTokenHashMap.put(str1, SMX.getStopWordToken1());
                }
                if (stopWordLemmaF == null) {
                    stopWordLemmaF = SMX.getStopWordLemmaF();
                }
                if (stopWordLemma1 == null) {
                    stopWordLemmaHashMap.put(str1, SMX.getStopWordLemma1());
                }
                if (PairCounterF == null) {
                    PairCounterF = SMX.getPairCounterF();
                }
                if (PairCounter1 == null) {
                    PairCounterHashMap.put(str1, SMX.getPairCounter1());
                }

                var getSMX: SimilarityMatrix = SMX.callSMX()
                val scoreRelationLastUserMsg = getSMX.distance
                if (scoreRelationLastUserMsg > preRelationUserCounters) {
                    preRelationUserCounters = scoreRelationLastUserMsg
                    concurrentRelations.add(getSMX.secondaryString)
                }
            }
        }
        val cacheRequirement = 6500;
        if (preRelationUserCounters > cacheRequirement && !values_copy.contains(strF) && filterContent(strF)) {
            DataMapper.InsertMYSQLStrings(strF)
            DataMapper.checkStringsToDelete();
        }
        val randomLenghtPermit = strF.length * (Math.random() * Math.random() * Math.random() * (Math.random() * 10))
        Collections.reverse(concurrentRelations)
        val mysqlUpdateLastUsed: ArrayList<String> = ArrayList()
        if (!concurrentRelations.isEmpty()) {
            for (secondaryRelation in concurrentRelations) {
                if (SB.toString().length > randomLenghtPermit && !SB.toString().isEmpty()) {
                    break
                }
                SB.append(secondaryRelation).append(" ")
                mysqlUpdateLastUsed.add(secondaryRelation)
            }
        }
        if (SB.toString().isEmpty()) {
            return "failure, preventing stuckness"
        }
        DataMapper.updateLastUsed(mysqlUpdateLastUsed);
        return SB.toString()
    }

    private fun getJMWEAnnotation(str1: String) {
        val jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str1)
        jmweAnnotationCache.put(str1, jmweAnnotation)
    }

    fun getResponseMsg(str: String, personName: String, stanfordCoreNLP: StanfordCoreNLP,
                       stanfordCoreNLPSentiment: StanfordCoreNLP, ingameResponse: Boolean): String {
        var responseFutures: String = ""
        runBlocking {
            launch(Dispatchers.Default) {
                var strF = trimString(str)
                responseFutures = getResponseFutures(strF, stanfordCoreNLP, stanfordCoreNLPSentiment)
                if (!ingameResponse) {
                    responseFutures = checkPersonPresentInSentence(personName, responseFutures, strF, stanfordCoreNLP,
                            stanfordCoreNLPSentiment)
                }
                yield()
            }.join()
        }
        return responseFutures
    }

    private fun checkPersonPresentInSentence(personName: String, responseMsg: String, userLastMessage: String,
                                             stanfordCoreNLP: StanfordCoreNLP,
                                             stanfordCoreNLPSentiment: StanfordCoreNLP): String {
        try {
            val pipelineCoreDcoument = CoreDocument(responseMsg)
            val pipelineCoreDcoumentLastMsg = CoreDocument(userLastMessage)
            stanfordCoreNLP.annotate(pipelineCoreDcoument)
            stanfordCoreNLPSentiment.annotate(pipelineCoreDcoumentLastMsg)
            val regex = "(.*?\\d){10,}"
            for (em in pipelineCoreDcoument.entityMentions()) {
                val entityType = em.entityType()
                if (entityType == "PERSON") {
                    var str = responseMsg
                    val emText = em.text()
                    val pattern = Pattern.compile(regex)
                    val matcher = pattern.matcher(personName)
                    val isMatched = matcher.matches()
                    if (emText != personName && !isMatched) {
                        for (emLastMsg in pipelineCoreDcoumentLastMsg.entityMentions()) {
                            if (emText != emLastMsg.text() && !Character.isDigit(emLastMsg.text().trim { it <= ' ' }[0])) {
                                str = (responseMsg.substring(0, responseMsg.indexOf(emText)) + " "
                                        + emLastMsg + " " + responseMsg.substring(responseMsg.indexOf(emText)))
                            }
                        }
                        str += " $personName"
                        return str
                    }
                }
            }
        } catch (e: Exception) {
            println("""SCUFFED JAYZ: ${e.message}""".trimIndent())
        }
        return responseMsg
    }

    fun filterContent(str: String): Boolean {
        if (!str.isEmpty() && str.length > 3) {
            var str1Local: String = str.trim();
            if (str1Local.length > 2 && !str1Local.startsWith("!")) {
                return true
            }
        }
        return false
    }

    fun getCoreDocumentsSuggested(pipeline: StanfordCoreNLP, str: String) {
        val annotation = Annotation(str)
        pipeline.annotate(annotation)
        val coreDocument = CoreDocument(annotation)
        coreDocumentAnnotationCache.put(str, coreDocument)
    }
}