projects-jenz/ArtificialAutism/src/main/java/FunctionLayer/Datahandler.java

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package FunctionLayer;

import DataLayer.DataMapper;
import FunctionLayer.StanfordParser.SentimentAnalyzerTest;
import FunctionLayer.StanfordParser.SentimentValueCache;
import com.google.common.base.Stopwatch;
import com.google.common.collect.MapMaker;
import edu.stanford.nlp.ie.AbstractSequenceClassifier;
import edu.stanford.nlp.ie.crf.CRFClassifier;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.CoreDocument;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
import edu.stanford.nlp.trees.GrammaticalStructureFactory;
import edu.stanford.nlp.trees.TreebankLanguagePack;
import java.io.IOException;
import static java.lang.Math.random;
import java.sql.SQLException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinTask;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;

/**
 *
 * @author install1
 */
public class Datahandler {

    public static final long EXPIRE_TIME_IN_SECONDS = TimeUnit.SECONDS.convert(10, TimeUnit.MINUTES);
    public static final long EXPIRE_TIME_IN_SECONDS1 = TimeUnit.SECONDS.convert(10, TimeUnit.HOURS);
    public static Datahandler instance = new Datahandler();
    private static Annotation strAnno;
    private static Annotation strAnnoSentiment;
    private static Annotation strAnnoJMWE;
    private static CoreDocument coreDoc;
    private static final ConcurrentMap<Integer, String> stringCache = new MapMaker().concurrencyLevel(6).makeMap();
    private static ConcurrentMap<String, Annotation> pipelineAnnotationCache;
    private static ConcurrentMap<String, Annotation> pipelineSentimentAnnotationCache;
    private static ConcurrentMap<String, Annotation> jmweAnnotationCache;
    private static ConcurrentMap<String, CoreDocument> coreDocumentAnnotationCache;
    private static ConcurrentMap<String, SentimentValueCache> sentimentCachingMap = new MapMaker().concurrencyLevel(6).makeMap();
    private LinkedHashMap<String, LinkedHashMap<String, Double>> lHMSMX = new LinkedHashMap();
    private final Stopwatch stopwatch;
    private static String similar = "";
    private static String shiftReduceParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz";
    private static String sentimentModel = "edu/stanford/nlp/models/sentiment/sentiment.ser.gz";
    private static String lexParserEnglishRNN = "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz";
    private static String taggerPath = "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger";
    private static String nerModel = "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz";
    private static String nerModel2 = "edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz";
    private static String nerModel3 = "edu/stanford/nlp/models/ner/english.muc.7class.caseless.distsim.crf.ser.gz";
    private static final String customStopWordList = "start,starts,period,periods,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with";
    private static MaxentTagger tagger;
    private static String[] options = {"-maxLength", "100"};
    private static Properties props = new Properties();
    private static Properties propsSentiment = new Properties();
    private static GrammaticalStructureFactory gsf;
    private static LexicalizedParser lp;
    private static TreebankLanguagePack tlp;
    private static AbstractSequenceClassifier<CoreLabel> classifier;
    // set up Stanford CoreNLP pipeline
    private static final StanfordCoreNLP pipeline = getPipeLineSetUp();
    private static StanfordCoreNLP pipelineSentiment;

    public Datahandler() {
        this.stopwatch = Stopwatch.createUnstarted();
        this.jmweAnnotationCache = new MapMaker().concurrencyLevel(3).makeMap();
        this.pipelineAnnotationCache = new MapMaker().concurrencyLevel(4).makeMap();
        this.pipelineSentimentAnnotationCache = new MapMaker().concurrencyLevel(4).makeMap();
        this.coreDocumentAnnotationCache = new MapMaker().concurrencyLevel(5).makeMap();
    }

    public static StanfordCoreNLP getPipeline() {
        return pipeline;
    }

    private static StanfordCoreNLP getPipeLineSetUp() {
        props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,parse");
        props.setProperty("parse.model", shiftReduceParserPath);
        props.setProperty("parse.maxlen", "90");
        props.setProperty("parse.binaryTrees", "true");
        props.setProperty("threads", "25");
        props.setProperty("pos.maxlen", "90");
        props.setProperty("tokenize.maxlen", "90");
        props.setProperty("ssplit.maxlen", "90");
        props.setProperty("lemma.maxlen", "90");
        props.setProperty("ner.model", nerModel + "," + nerModel2 + "," + nerModel3);
        props.setProperty("ner.combinationMode", "HIGH_RECALL");
        props.setProperty("regexner.ignorecase", "true");
        props.setProperty("ner.fine.regexner.ignorecase", "true");
        props.setProperty("tokenize.options", "untokenizable=firstDelete");
        return new StanfordCoreNLP(props);
    }

    public void shiftReduceParserInitiate() {
        //got 8 cores
        CountDownLatch cdl = new CountDownLatch(2);
        new Thread(() -> {
            try {
                classifier = CRFClassifier.getClassifierNoExceptions(nerModel);
            } catch (ClassCastException ex) {
                Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
            }
            cdl.countDown();
        }).start();
        new Thread(() -> {
            propsSentiment.setProperty("parse.model", lexParserEnglishRNN);
            propsSentiment.setProperty("sentiment.model", sentimentModel);
            propsSentiment.setProperty("parse.maxlen", "90");
            propsSentiment.setProperty("threads", "25");
            propsSentiment.setProperty("pos.maxlen", "90");
            propsSentiment.setProperty("tokenize.maxlen", "90");
            propsSentiment.setProperty("ssplit.maxlen", "90");
            propsSentiment.setProperty("annotators", "tokenize,ssplit,pos,parse,sentiment,lemma,stopword"); //coref too expensive memorywise
            propsSentiment.setProperty("customAnnotatorClass.stopword", "FunctionLayer.StopwordAnnotator");
            propsSentiment.setProperty(StopwordAnnotator.STOPWORDS_LIST, customStopWordList);
            propsSentiment.setProperty("tokenize.options", "untokenizable=firstDelete");
            pipelineSentiment = new StanfordCoreNLP(propsSentiment);
            tagger = new MaxentTagger(taggerPath);
            cdl.countDown();
        }).start();
        lp = LexicalizedParser.loadModel(lexParserEnglishRNN, options);
        tlp = lp.getOp().langpack();
        gsf = tlp.grammaticalStructureFactory();
        try {
            cdl.await();
        } catch (InterruptedException ex) {
            //System.out.println("cdl await interrupted: " + ex.getLocalizedMessage() + "\n");
        }
        System.out.println("finished shiftReduceParserInitiate\n");
    }

    public static AbstractSequenceClassifier<CoreLabel> getClassifier() {
        return classifier;
    }

    public static void setClassifier(AbstractSequenceClassifier<CoreLabel> classifier) {
        Datahandler.classifier = classifier;
    }

    public void updateStringCache() {
        try {
            checkIfUpdateStrings(true);
        } catch (CustomError ex) {
            Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    public static GrammaticalStructureFactory getGsf() {
        return gsf;
    }

    public static MaxentTagger getTagger() {
        return tagger;
    }

    private Map<Integer, String> getCache() throws SQLException, IOException, CustomError {
        return DataMapper.getAllStrings();
    }

    public int getlHMSMXSize() {
        return lHMSMX.size();
    }

    public int getstringCacheSize() {
        return stringCache.size();
    }

    public void initiateMYSQL() throws SQLException, IOException {
        try {
            DataMapper.createTables();
            stringCache.putAll(getCache());
            // lHMSMX = DataMapper.getAllRelationScores();
        } catch (CustomError ex) {
            Logger.getLogger(Datahandler.class
                    .getName()).log(Level.SEVERE, null, ex);
        }
    }

    public void addHLstatsMessages() {
        ConcurrentMap<String, Integer> hlStatsMessages = new MapMaker().concurrencyLevel(2).makeMap();
        ConcurrentMap<Integer, String> strCacheLocal = stringCache;
        Collection<String> strs = DataMapper.getHLstatsMessages().values();
        for (String str : strs) {
            if (hlStatsMessages.get(str) == null) {
                hlStatsMessages.put(str, hlStatsMessages.size());
            }
        }
        int capacity = 1250;
        hlStatsMessages.keySet().forEach(str -> {
            if (!str.startsWith("!") && MessageResponseHandler.getStr().values().size() < capacity) {
                String orElse = strCacheLocal.values().parallelStream().filter(e -> e.equals(str)).findAny().orElse(null);
                if (orElse == null) {
                    MessageResponseHandler.getMessage(str);
                }
            }
        });
    }

    public void instantiateAnnotationMapJMWE() {
        if (!stringCache.isEmpty()) {
            ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(stringCache.values());
            for (Entry<String, Annotation> entries : jmweAnnotation.entrySet()) {
                jmweAnnotationCache.put(entries.getKey(), entries.getValue());
            }
        }
    }

    public void instantiateAnnotationMap() {
        if (!stringCache.isEmpty()) {
            ConcurrentMap<String, Annotation> Annotationspipeline = new MapMaker().concurrencyLevel(2).makeMap();
            ConcurrentMap<String, Annotation> AnnotationspipelineSentiment = new MapMaker().concurrencyLevel(2).makeMap();
            stringCache.values().parallelStream().forEach(str -> {
                Annotation strAnno = new Annotation(str);
                strAnno.compact();
                Annotationspipeline.put(str, strAnno);
                Annotation strAnno2 = new Annotation(str);
                strAnno2.compact();
                AnnotationspipelineSentiment.put(str, strAnno2);
            });
            ConcurrentMap<String, CoreDocument> coreDocumentpipelineMap = getMultipleCoreDocumentsWaySuggestion(stringCache.values(), pipeline);
            pipeline.annotate(Annotationspipeline.values());
            pipelineSentiment.annotate(AnnotationspipelineSentiment.values());
            Annotationspipeline.entrySet().forEach(pipelineEntry -> {
                //relatively experimental change
                pipelineEntry.getValue().compact();
                pipelineAnnotationCache.put(pipelineEntry.getKey(), pipelineEntry.getValue());
            });
            AnnotationspipelineSentiment.entrySet().forEach(pipelineEntry -> {
                pipelineEntry.getValue().compact();
                pipelineSentimentAnnotationCache.put(pipelineEntry.getKey(), pipelineEntry.getValue());
            });
            coreDocumentpipelineMap.entrySet().stream().forEach(CD -> {
                coreDocumentAnnotationCache.put(CD.getKey(), CD.getValue());
            });
        }
    }

    private ConcurrentMap<Integer, String> futuresReturnOverallEvaluation(List<SimilarityMatrix> similarityMatrixes) {
        ConcurrentMap<Integer, String> strmapreturn = new MapMaker().concurrencyLevel(6).makeMap();
        if (!similarityMatrixes.isEmpty()) {
            String newPrimary = similarityMatrixes.get(0).getPrimaryString();
            int evaluationCap = 500;
            int iterator = 0;
            for (SimilarityMatrix SMX : similarityMatrixes) {
                final Double scoreRelationNewMsgToRecentMsg = SMX.getDistance();
                if (scoreRelationNewMsgToRecentMsg > evaluationCap) {
                    strmapreturn = addSMXToMapReturn(strmapreturn, SMX);
                }
                //System.out.println("similarityMatrixes size: " + similarityMatrixes.size() + "\niterator: " + iterator);
                iterator++;
            }
        }
        return strmapreturn;
    }

    private ConcurrentMap<Integer, String> addSMXToMapReturn(ConcurrentMap<Integer, String> strmapreturn, SimilarityMatrix SMX) {
        if (!strmapreturn.containsValue(SMX.getPrimaryString())) {
            strmapreturn.put(strmapreturn.size(), SMX.getPrimaryString());
            String transmittedStr = SMX.getSecondaryString();
            SentimentValueCache cacheValue1 = SMX.getCacheValue1();
            SentimentValueCache cacheValue2 = SMX.getCacheValue2();
            if (cacheValue1 != null && !sentimentCachingMap.keySet().contains(SMX.getPrimaryString())) {
                sentimentCachingMap.put(SMX.getSecondaryString(), SMX.getCacheValue1());
            }
            if (cacheValue2 != null && !sentimentCachingMap.keySet().contains(transmittedStr)) {
                sentimentCachingMap.put(transmittedStr, SMX.getCacheValue2());
            }
        }
        return strmapreturn;
    }

    private List<SimilarityMatrix> StrComparringNoSentenceRelationMap(
            ConcurrentMap<Integer, String> strCacheLocal, Collection<String> str, ConcurrentMap<String, Annotation> localJMWEMap,
            ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
            ConcurrentMap<String, CoreDocument> localCoreDocumentMap, CompletionService<SimilarityMatrix> ecs, int index) {
        int prefix_size = 125;
        SentimentValueCache sentimentCacheStr = sentimentCachingMap.getOrDefault(str, null);
        List<SimilarityMatrix> smxReturnList = new ArrayList();
        List<String> randomIterationComparision = new ArrayList();
        int iteratecap = strCacheLocal.size() > prefix_size ? strCacheLocal.size() - prefix_size : strCacheLocal.size();
        int iterator = ThreadLocalRandom.current().nextInt(0, iteratecap);
        int iterated = 0;
        for (String str1 : strCacheLocal.values()) {
            if (iterated >= iterator && iterated < iterator + prefix_size) {
                randomIterationComparision.add(str1);
            }
            if (iterated > iterator + prefix_size) {
                break;
            }
            iterated++;
        }
        for (String str1 : randomIterationComparision) {
            for (String str2 : str) {
                if (!str2.equals(str1)) {
                    SimilarityMatrix SMXInit = new SimilarityMatrix(str2, str1);
                    SentimentValueCache sentimentCacheStr1 = sentimentCachingMap.getOrDefault(str1, null);
                    Callable<SimilarityMatrix> worker;
                    if (stringCache.size() < prefix_size) {
                        worker = new SentimentAnalyzerTest(str2, str1, SMXInit,
                                localJMWEMap.get(str), localJMWEMap.get(str1), localPipelineAnnotation.get(str),
                                localPipelineAnnotation.get(str1), localPipelineSentimentAnnotation.get(str),
                                localPipelineSentimentAnnotation.get(str1), localCoreDocumentMap.get(str), localCoreDocumentMap.get(str1), sentimentCacheStr, sentimentCacheStr1);
                    } else {
                        worker = new SentimentAnalyzerTest(str2, str1, SMXInit,
                                localJMWEMap.get(str), jmweAnnotationCache.get(str1), localPipelineAnnotation.get(str),
                                pipelineAnnotationCache.get(str1), localPipelineSentimentAnnotation.get(str),
                                pipelineSentimentAnnotationCache.get(str1), localCoreDocumentMap.get(str), coreDocumentAnnotationCache.get(str1), sentimentCacheStr, sentimentCacheStr1);
                    }
                    ecs.submit(worker);
                    index++;
                }
            }
        }
        for (int i = 0; i < index; i++) {
            try {
                Future<SimilarityMatrix> take = ecs.take();
                SimilarityMatrix smx = take.get();
                if (smx != null && !smxReturnList.contains(smx)) {
                    smxReturnList.add(smx);
                }
            } catch (InterruptedException | ExecutionException ex) {
                //
            }
        }
        index = 0;
        System.out.println("smxReturnList size: " + smxReturnList.size());
        return smxReturnList;
    }

    private ConcurrentMap<Integer, String> stringIteratorComparator(ConcurrentMap<Integer, String> strmap,
            ConcurrentMap<Integer, String> strCacheLocal, ConcurrentMap<String, Annotation> localJMWEMap,
            ConcurrentMap<String, Annotation> localPipelineAnnotation, ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation,
            ConcurrentMap<String, CoreDocument> localCoreDocumentMap) {
        ExecutorService threadPool = Executors.newCachedThreadPool();
        CompletionService<SimilarityMatrix> ecs = new ExecutorCompletionService<>(threadPool);
        int index = 0;
        //System.out.println("strmap siuze: " + strmap.size());
        List<SimilarityMatrix> StrComparringNoSentenceRelationMap = StrComparringNoSentenceRelationMap(strCacheLocal, strmap.values(),
                localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap, ecs, index);
        threadPool.shutdown();
        //System.out.println("StrComparringNoSentenceRelationMap size: " + StrComparringNoSentenceRelationMap.size());
        Collections.sort(StrComparringNoSentenceRelationMap, (e1, e2) -> e1.getPrimaryString().compareTo(e2.getPrimaryString()));
        ConcurrentMap<Integer, String> strmapreturn = futuresReturnOverallEvaluation(StrComparringNoSentenceRelationMap);
        //System.out.println("strmapreturn size: " + strmapreturn.size());
        return strmapreturn;
    }

    private ConcurrentMap<Integer, String> removeNonSensicalStrings(ConcurrentMap<Integer, String> strmap) {
        final ConcurrentMap<Integer, String> strCacheLocal = stringCache.size() < 150 ? strmap : stringCache;
        final ConcurrentMap<String, Annotation> localJMWEMap = getMultipleJMWEAnnotation(strmap.values());
        final ConcurrentMap<String, Annotation> localPipelineAnnotation = getMultiplePipelineAnnotation(strmap.values());
        final ConcurrentMap<String, Annotation> localPipelineSentimentAnnotation = getMultiplePipelineSentimentAnnotation(strmap.values());
        final ConcurrentMap<String, CoreDocument> localCoreDocumentMap = getMultipleCoreDocumentsWaySuggestion(strmap.values(), pipeline);
        return stringIteratorComparator(strmap, strCacheLocal, localJMWEMap, localPipelineAnnotation, localPipelineSentimentAnnotation, localCoreDocumentMap);
    }

    public synchronized void checkIfUpdateStrings(boolean hlStatsMsg) throws CustomError {
        if (stopwatch.elapsed(TimeUnit.SECONDS) >= EXPIRE_TIME_IN_SECONDS || !stopwatch.isRunning()) {
            ConcurrentMap<Integer, String> str = MessageResponseHandler.getStr();
            System.out.println("str size: " + str.size());
            str = cutContent(str, hlStatsMsg);
            str = filterContent(str);
            str = removeSlacks(str);
            //System.out.println("finished removeSlacks \n" + str.size() + "\n");
            str = removeNonSensicalStrings(str);
            System.out.println("removeNonSensicalStrings str size POST: " + str.size() + "\n");
            str = annotationCacheUpdate(str);
            System.out.println("annotationCacheUpdate str size POST: " + str.size() + "\n");
            ConcurrentMap<Integer, String> strf = str;
            if (!stringCache.isEmpty()) {
                new Thread(() -> {
                    try {
                        DataMapper.InsertMYSQLStrings(strf);
                    } catch (CustomError ex) {
                        Logger.getLogger(Datahandler.class
                                .getName()).log(Level.SEVERE, null, ex);
                    }
                    MessageResponseHandler.setStr(new MapMaker().concurrencyLevel(6).makeMap());
                }).start();
            } else {
                try {
                    DataMapper.InsertMYSQLStrings(strf);
                } catch (CustomError ex) {
                    Logger.getLogger(Datahandler.class
                            .getName()).log(Level.SEVERE, null, ex);
                }
                MessageResponseHandler.setStr(new MapMaker().concurrencyLevel(2).makeMap());
            }
            if (!stopwatch.isRunning()) {
                stopwatch.start();
            } else {
                stopwatch.reset();
            }
        }
    }

    private String trimString(String str) {
        str = str.trim();
        if (str.startsWith("<@")) {
            str = str.substring(str.indexOf("> ") + 2);
        }
        return str;
    }

    private String getResponseFutures(String strF) {
        List<String> values_copy = new ArrayList<String>(stringCache.values());
        int maxsize = values_copy.size() > 500 ? 500 : values_copy.size();
        Collections.shuffle(values_copy);
        List<String> strCache = values_copy.subList(0, maxsize);
        double preRelationUserCounters = -150000.0;
        //WHY THE FUCK CANT YOU JUST TRANSFER A SimilarityMatrix OBJECT LIST LIKE ANY OTHER NORMAL COLLECTION, WHY DOES IT HAVE TO BE A FUCKING STRING LIST
        List<String> concurrentRelations = new ArrayList();
        List<Callable<SimilarityMatrix>> call_able_list = new ArrayList();
        for (String str1 : strCache) {
            if (!strF.equals(str1)) {
                SentimentValueCache sentimentCacheStr1 = sentimentCachingMap.getOrDefault(str1, null);
                Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(strF, str1, new SimilarityMatrix(strF, str1),
                        strAnnoJMWE, jmweAnnotationCache.get(str1), strAnno,
                        pipelineAnnotationCache.get(str1), strAnnoSentiment,
                        pipelineSentimentAnnotationCache.get(str1), coreDoc, coreDocumentAnnotationCache.get(str1), null, sentimentCacheStr1);
                call_able_list.add(worker);
            }
        }
        for (Callable<SimilarityMatrix> callSMX : call_able_list) {
            try {
                SimilarityMatrix getSMX = callSMX.call();
                if (getSMX != null) {
                    Double scoreRelationLastUserMsg = getSMX.getDistance();
                    if (scoreRelationLastUserMsg > preRelationUserCounters) {
                        preRelationUserCounters = scoreRelationLastUserMsg;
                        concurrentRelations.add(getSMX.getSecondaryString());
                        //System.out.println("secondary: " + getSMX.getSecondaryString() + "\nDistance: " + getSMX.getDistance() + "\n");
                        //System.out.println("SUCESS concurrentRelationsMap size: " + concurrentRelations.size() + "\n");
                    }
                }
            } catch (Exception ex) {
                Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
        StringBuilder SB = new StringBuilder();
        double randomLenghtPermit = strF.length() * (Math.random() * Math.random() * Math.random());
        Collections.reverse(concurrentRelations);
        if (concurrentRelations.isEmpty()) {
            return "failure, preventing stuckness";
        }
        String firstRelation = concurrentRelations.get(0);
        for (String secondaryRelation : concurrentRelations) {
            if (SB.toString().length() > randomLenghtPermit && !SB.toString().isEmpty()) {
                break;
            }
            boolean append = appendToString(firstRelation, secondaryRelation);
            if (append) {
                SB.append(secondaryRelation).append(" ");
            }
        }
        return SB.toString();
    }

    private boolean appendToString(String firstRelation, String secondaryRelation) {
        if (firstRelation.equals(secondaryRelation)) {
            return true;
        }
        Double scoreRelationStrF = getScoreRelationStrF(firstRelation, secondaryRelation);
        if (scoreRelationStrF > 1900) {
            return true;
        }
        return false;
    }

    public String getResponseMsg(String str) throws CustomError {
        String strF = trimString(str);
        getSingularAnnotation(strF);
        return getResponseFutures(strF);
    }

    public void getSingularAnnotation(String str) {
        strAnno = new Annotation(str);
        strAnno.compact();
        pipeline.annotate(strAnno);
        strAnnoSentiment = new Annotation(str);
        strAnnoSentiment.compact();
        pipelineSentiment.annotate(strAnnoSentiment);
        List<String> notactualList = new ArrayList();
        notactualList.add(str);
        ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(notactualList);
        strAnnoJMWE = jmweAnnotation.values().iterator().next();
        strAnnoJMWE.compact();
        CoreDocument coreDocument = new CoreDocument(str);
        pipeline.annotate(coreDocument);
        coreDoc = coreDocument;
    }

    private static ConcurrentMap<String, Annotation> getMultipleJMWEAnnotation(Collection<String> str) {
        ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(str);
        return jmweAnnotation;
    }

    private static ConcurrentMap<String, Annotation> getMultiplePipelineAnnotation(Collection<String> str) {
        ConcurrentMap<String, Annotation> pipelineAnnotationMap = new MapMaker().concurrencyLevel(2).makeMap();
        for (String str1 : str) {
            Annotation strAnno1 = new Annotation(str1);
            pipelineAnnotationMap.put(str1, strAnno1);
        }
        pipeline.annotate(pipelineAnnotationMap.values());
        return pipelineAnnotationMap;
    }

    private static ConcurrentMap<String, Annotation> getMultiplePipelineSentimentAnnotation(Collection<String> str) {
        ConcurrentMap<String, Annotation> pipelineAnnotationMap = new MapMaker().concurrencyLevel(2).makeMap();
        for (String str1 : str) {
            Annotation strAnno1 = new Annotation(str1);
            pipelineAnnotationMap.put(str1, strAnno1);
        }
        pipelineSentiment.annotate(pipelineAnnotationMap.values());
        return pipelineAnnotationMap;
    }

    private Double getScoreRelationNewMsgToRecentMsg(String str, String mostRecentMsg) {
        SimilarityMatrix SMX = new SimilarityMatrix(str, mostRecentMsg);
        SentimentValueCache cacheSentiment1 = sentimentCachingMap.getOrDefault(str, null);
        SentimentValueCache cacheSentiment2 = sentimentCachingMap.getOrDefault(mostRecentMsg, null);
        Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, mostRecentMsg, SMX,
                jmweAnnotationCache.get(str), jmweAnnotationCache.get(mostRecentMsg), pipelineAnnotationCache.get(str),
                pipelineAnnotationCache.get(mostRecentMsg), pipelineSentimentAnnotationCache.get(str),
                pipelineSentimentAnnotationCache.get(mostRecentMsg), coreDocumentAnnotationCache.get(str),
                coreDocumentAnnotationCache.get(mostRecentMsg), cacheSentiment1, cacheSentiment2);
        SimilarityMatrix callSMX = null;
        try {
            callSMX = worker.call();
        } catch (Exception ex) {
            Logger.getLogger(Datahandler.class
                    .getName()).log(Level.SEVERE, null, ex);
        }
        if (callSMX != null) {
            double smxDistance = callSMX.getDistance();
            return smxDistance;
        }
        return 0.0;
    }

    private Double getScoreRelationStrF(String str, String mostRecentMsg) {
        SimilarityMatrix SMX = new SimilarityMatrix(str, mostRecentMsg);
        SentimentValueCache cacheSentiment1 = sentimentCachingMap.getOrDefault(str, null);
        SentimentValueCache cacheSentiment2 = sentimentCachingMap.getOrDefault(mostRecentMsg, null);
        Callable<SimilarityMatrix> worker = new SentimentAnalyzerTest(str, mostRecentMsg, SMX,
                strAnnoJMWE, jmweAnnotationCache.get(mostRecentMsg), strAnno,
                pipelineAnnotationCache.get(mostRecentMsg), strAnnoSentiment,
                pipelineSentimentAnnotationCache.get(mostRecentMsg), coreDoc, coreDocumentAnnotationCache.get(mostRecentMsg), cacheSentiment1, cacheSentiment2);
        SimilarityMatrix callSMX = null;
        try {
            callSMX = worker.call();
        } catch (Exception ex) {
            Logger.getLogger(Datahandler.class
                    .getName()).log(Level.SEVERE, null, ex);
        }
        if (callSMX != null) {
            double smxDistance = callSMX.getDistance();
            return smxDistance;
        }
        return 0.0;
    }

    public static ConcurrentMap<Integer, String> cutContent(ConcurrentMap<Integer, String> str, boolean hlStatsMsg) {
        ConcurrentMap<Integer, String> returnlist = new MapMaker().concurrencyLevel(2).makeMap();
        str.values().forEach(str1 -> {
            int iend = str1.indexOf("content: ");
            if (iend != -1) {
                String trs = str1.substring(iend + 9);
                returnlist.put(returnlist.size() + 1, trs.substring(0, trs.length() - 1));
            } else if (hlStatsMsg) {
                returnlist.put(returnlist.size() + 1, str1);
            }
        });
        return returnlist;
    }

    public static ConcurrentMap<Integer, String> filterContent(ConcurrentMap<Integer, String> str) {
        ConcurrentMap<Integer, String> strlistreturn = new MapMaker().concurrencyLevel(2).makeMap();
        str.values().forEach(str1 -> {
            if (!str1.isEmpty() && str1.length() > 3) {
                str1 = str1.trim();
                if (str1.contains("PM*")) {
                    str1 = str1.substring(str1.indexOf("PM*") + 3);
                }
                if (str1.contains("AM*")) {
                    str1 = str1.substring(str1.indexOf("AM*") + 3);
                }
                for (Character c : str1.toCharArray()) {
                    if (c == '?' || c == '°') {
                        str1 = str1.replace("?", " <:wlenny:514861023002624001> ");
                        str1 = str1.replace("°", " <:wlenny:514861023002624001> ");
                    }
                }
                if (str1.contains("(Counter-Terrorist)")) {
                    str1 = str1.replace("(Counter-Terrorist)", " ");
                }
                if (str1.contains("(Terrorist)")) {
                    str1 = str1.replace("(Terrorist)", " ");
                }
                if (str1.contains("(Spectator)")) {
                    str1 = str1.replace("(Spectator)", " ");
                }
                if (str1.contains("*DEAD*")) {
                    str1 = str1.replace("*DEAD*", " ");
                }
                if (str1.contains("{red}")) {
                    str1 = str1.replace("{red}", " ");
                }
                if (str1.contains("{orange}")) {
                    str1 = str1.replace("{orange}", " ");
                }
                if (str1.contains("{yellow}")) {
                    str1 = str1.replace("{yellow}", " ");
                }
                if (str1.contains("{green}")) {
                    str1 = str1.replace("{green}", " ");
                }
                if (str1.contains("{lightblue}")) {
                    str1 = str1.replace("{lightblue}", " ");
                }
                if (str1.contains("{blue}")) {
                    str1 = str1.replace("{blue}", " ");
                }
                if (str1.contains("{purple}")) {
                    str1 = str1.replace("{purple}", " ");
                }
                if (str1.contains("{white}")) {
                    str1 = str1.replace("{white}", " ");
                }
                if (str1.contains("{fullblue}")) {
                    str1 = str1.replace("{fullblue}", " ");
                }
                if (str1.contains("{cyan}")) {
                    str1 = str1.replace("{cyan}", " ");
                }
                if (str1.contains("{lime}")) {
                    str1 = str1.replace("{lime}", " ");
                }
                if (str1.contains("{deeppink}")) {
                    str1 = str1.replace("{deeppink}", " ");
                }
                if (str1.contains("{slategray}")) {
                    str1 = str1.replace("{slategray}", " ");
                }
                if (str1.contains("{dodgerblue}")) {
                    str1 = str1.replace("{dodgerblue}", " ");
                }
                if (str1.contains("{black}")) {
                    str1 = str1.replace("{black}", " ");
                }
                if (str1.contains("{orangered}")) {
                    str1 = str1.replace("{orangered}", " ");
                }
                if (str1.contains("{darkorchid}")) {
                    str1 = str1.replace("{darkorchid}", " ");
                }
                if (str1.contains("{pink}")) {
                    str1 = str1.replace("{pink}", " ");
                }
                if (str1.contains("{lightyellow}")) {
                    str1 = str1.replace("{lightyellow}", " ");
                }
                if (str1.contains("{chocolate}")) {
                    str1 = str1.replace("{chocolate}", " ");
                }
                if (str1.contains("{beige}")) {
                    str1 = str1.replace("{beige}", " ");
                }
                if (str1.contains("{azure}")) {
                    str1 = str1.replace("{azure}", " ");
                }
                if (str1.contains("{yellowgreen}")) {
                    str1 = str1.replace("{yellowgreen}", " ");
                }
                str1 = str1.trim();
                if (str1.length() > 2 && (!str1.startsWith("!"))) {
                    strlistreturn.put(strlistreturn.size() + 1, str1);
                }
            }
        });
        return strlistreturn;
    }

    private ConcurrentMap<Integer, String> removeSlacks(ConcurrentMap<Integer, String> str) {
        ConcurrentMap<Integer, String> strreturn = new MapMaker().concurrencyLevel(2).makeMap();
        if (stringCache.isEmpty()) {
            return str;
        }
        Collection<String> values = stringCache.values();
        str.values().forEach(str1 -> {
            boolean tooclosematch = false;
            for (String strVals : values) {
                LevenshteinDistance leven = new LevenshteinDistance(strVals, str1);
                double Distance = leven.computeLevenshteinDistance();
                Double maxpermittedDistance = 2.5;
                if (Distance < maxpermittedDistance) {
                    tooclosematch = true;
                    break;
                }
            }
            if (!tooclosematch) {
                strreturn.put(strreturn.size() + 1, str1);
                //System.out.println("adding strreturn str1: " + str1 + "\n");
            }
        });
        return strreturn;
    }

    private ConcurrentMap<Integer, String> annotationCacheUpdate(ConcurrentMap<Integer, String> strmap) {
        ConcurrentMap<String, Annotation> jmweAnnotation = PipelineJMWESingleton.INSTANCE.getJMWEAnnotation(strmap.values());
        for (Entry<String, Annotation> jmweitr : jmweAnnotation.entrySet()) {
            jmweAnnotationCache.put(jmweitr.getKey(), jmweitr.getValue());
        }
        ConcurrentMap<String, Annotation> Annotationspipeline = new MapMaker().concurrencyLevel(4).makeMap();
        ConcurrentMap<String, Annotation> AnnotationspipelineSentiment = new MapMaker().concurrencyLevel(4).makeMap();
        ConcurrentMap<String, CoreDocument> coreDocumentpipelineMap = getMultipleCoreDocumentsWaySuggestion(strmap.values(), pipeline);
        strmap.values().forEach(str -> {
            Annotation strAnno1 = new Annotation(str);
            Annotationspipeline.put(str, strAnno1);
            Annotation strAnno2 = new Annotation(str);
            AnnotationspipelineSentiment.put(str, strAnno2);
            stringCache.put(stringCache.size() + 1, str);
        });
        pipeline.annotate(Annotationspipeline.values());
        pipelineSentiment.annotate(AnnotationspipelineSentiment.values());
        Annotationspipeline.entrySet().forEach(pipelineEntry -> {
            if (pipelineEntry != null) {
                pipelineAnnotationCache.put(pipelineEntry.getKey(), pipelineEntry.getValue());
            }
        });
        AnnotationspipelineSentiment.entrySet().forEach(pipelineEntry -> {
            if (pipelineEntry != null) {
                pipelineSentimentAnnotationCache.put(pipelineEntry.getKey(), pipelineEntry.getValue());
            }
        });
        coreDocumentpipelineMap.entrySet().forEach(coreDocumentEntry -> {
            coreDocumentAnnotationCache.put(coreDocumentEntry.getKey(), coreDocumentEntry.getValue());
        });
        return strmap;
    }

    public int getMessageOverHead() {
        return stringCache.values().size() - (stringCache.values().size() / 10);
    }

    private static class AnnotationCollector<T> implements Consumer<T> {

        private static int i = 0;
        private List<T> annotationsT = new ArrayList();

        @Override
        public void accept(T ann) {
            //System.out.println("adding ann: " + ann.toString());
            annotationsT.add(ann);
        }
    }

    public static ConcurrentMap<String, CoreDocument> getMultipleCoreDocumentsWaySuggestion(Collection<String> str, StanfordCoreNLP localNLP) {
        AnnotationCollector<Annotation> annCollector = new AnnotationCollector();
        for (String exampleString : str) {
            localNLP.annotate(new Annotation(exampleString), annCollector);
            annCollector.i++;
            //System.out.println("iterator: " + annCollector.i + "\nstr size: " + str.size() + "\n");
        }
        try {
            Thread.sleep(8000);
        } catch (InterruptedException ex) {
            Logger.getLogger(Datahandler.class.getName()).log(Level.SEVERE, null, ex);
        }
        ConcurrentMap<String, CoreDocument> annotationreturnMap = new MapMaker().concurrencyLevel(6).makeMap();
        for (Annotation ann : annCollector.annotationsT) {
            if (ann != null) {
                ann.compact();
                CoreDocument CD = new CoreDocument(ann);
                annotationreturnMap.put(CD.text(), CD);
                //System.out.println("CD text:" + CD.text() + "\niterator: " + iterator + "\nsize: " + annCollector.annotationsT.size());
            }
        }
        return annotationreturnMap;
    }
}