/*
 ******************************************************************************
 * Copyright (C) 2005-2012, International Business Machines Corporation and        *
 * others. All Rights Reserved.                                               *
 ******************************************************************************
 */
package org.unicode.cldr.test;

import java.util.BitSet;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype;
import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.CLDRFile.Status;
import org.unicode.cldr.util.Factory;
import org.unicode.cldr.util.InternalCldrException;
import org.unicode.cldr.util.LocaleIDParser;
import org.unicode.cldr.util.PatternCache;
import org.unicode.cldr.util.PatternPlaceholders;
import org.unicode.cldr.util.PatternPlaceholders.PlaceholderStatus;
import org.unicode.cldr.util.SupplementalDataInfo;
import org.unicode.cldr.util.SupplementalDataInfo.BasicLanguageData;
import org.unicode.cldr.util.SupplementalDataInfo.BasicLanguageData.Type;
import org.unicode.cldr.util.SupplementalDataInfo.CurrencyDateInfo;
import org.unicode.cldr.util.UnicodeSetPrettyPrinter;
import org.unicode.cldr.util.XMLSource;
import org.unicode.cldr.util.XPathParts;

import com.ibm.icu.impl.Relation;
import com.ibm.icu.lang.UScript;
import com.ibm.icu.text.Collator;
import com.ibm.icu.text.DateTimePatternGenerator;
import com.ibm.icu.text.Normalizer2;
import com.ibm.icu.text.PluralRules;
import com.ibm.icu.text.Transform;
import com.ibm.icu.text.UnicodeSet;
import com.ibm.icu.util.ULocale;

public class CheckForExemplars extends FactoryCheckCLDR {
    private static final String STAND_IN = "#";

    // private final UnicodeSet commonAndInherited = new UnicodeSet(CheckExemplars.Allowed).complement();
    // "[[:script=common:][:script=inherited:][:alphabetic=false:]]");
    static String[] EXEMPLAR_SKIPS = {
        "/currencySpacing",
        "/exemplarCharacters",
        // "/pattern",
        "/localizedPatternChars",
        "/segmentations",
        "/references",
        "/localeDisplayNames/variants/",
        "/commonlyUsed",
        "/defaultNumberingSystem",
        "/otherNumberingSystems",
        "/exponential",
        "/nan",
        "/scientificFormats",
        "/inText",
        "/orientation",
        "/symbol[@alt=\"narrow\"]"
    };

    static String[] DATE_PARTS = {
        "/hourFormat",
        "/dateFormatItem",
        "/intervalFormatItem",
        "/dateFormatLength",
        "timeFormatLength"
    };

    static final UnicodeSet START_PAREN = new UnicodeSet("[[:Ps:]]").freeze();
    static final UnicodeSet END_PAREN = new UnicodeSet("[[:Pe:]]").freeze();
    static final UnicodeSet ALL_CURRENCY_SYMBOLS = new UnicodeSet("[[:Sc:]]").freeze();
    static final UnicodeSet LETTER = new UnicodeSet("[[A-Za-z]]").freeze();
    static final UnicodeSet NUMBERS = new UnicodeSet("[[:N:]]").freeze();
    static final UnicodeSet DISALLOWED_HOUR_FORMAT = new UnicodeSet("[[:letter:]]").remove('H').remove('m').freeze();
    static final UnicodeSet DISALLOWED_IN_RANGE = new UnicodeSet("[:L:]").freeze();

    private UnicodeSet exemplars;
    private UnicodeSet exemplarsPlusAscii;
    //private static final UnicodeSet DISALLOWED_IN_scriptRegionExemplars = new UnicodeSet("[()（）;,；，]").freeze();
    //private static final UnicodeSet DISALLOWED_IN_scriptRegionExemplarsWithParens = new UnicodeSet("[;,；，]").freeze();

    // Hack until cldrbug 6566 is fixed. TODO
    private static final Pattern IGNORE_PLACEHOLDER_PARENTHESES = PatternCache.get("\\p{Ps}#\\p{Pe}");

    // private UnicodeSet currencySymbolExemplars;
    private boolean skip;
    private Collator col;
    private Collator spaceCol;
    UnicodeSetPrettyPrinter prettyPrint;
    private Status otherPathStatus = new Status();
    private Matcher patternMatcher = ExampleGenerator.PARAMETER.matcher("");

    // for extracting date pattern text
    private DateTimePatternGenerator.FormatParser formatParser = new DateTimePatternGenerator.FormatParser();
    StringBuilder justText = new StringBuilder();

    // public static final Pattern SUPPOSED_TO_BE_MESSAGE_FORMAT_PATTERN = PatternCache.get("/(" +
    // "codePattern" +
    // "|dateRangePattern" +
    // "|dateTimeFormat[^/]*?/pattern" +
    // "|appendItem" +
    // "|intervalFormatFallback" +
    // "|hoursFormat" +
    // "|gmtFormat" +
    // "|regionFormat" +
    // "|fallbackFormat" +
    // "|unitPattern.*@count=\"(zero|one|two|few|many|other)\"" +
    // "|localePattern" +
    // "|localeKeyTypePattern" +
    // "|listPatternPart" +
    // "|ellipsis" +
    // "|monthPattern" +
    // ")");
    // private Matcher supposedToBeMessageFormat = SUPPOSED_TO_BE_MESSAGE_FORMAT_PATTERN.matcher("");

    public static final Pattern LEAD_OR_TRAIL_WHITESPACE_OK = PatternCache.get("/(" +
        "references/reference" +
        "|insertBetween" +
        ")");
    private Matcher leadOrTrailWhitespaceOk = LEAD_OR_TRAIL_WHITESPACE_OK.matcher("");

    private static UnicodeSet ASCII = (UnicodeSet) new UnicodeSet("[\\u0020-\\u007F]").freeze();

    private PatternPlaceholders patternPlaceholders = PatternPlaceholders.getInstance();
    private SupplementalDataInfo sdi;
    private Relation scriptToCurrencies;

    public CheckForExemplars(Factory factory) {
        super(factory);
        // patternPlaceholders = RegexLookup.of(new PlaceholderTransform())
        // .loadFromFile(PatternPlaceholders.class, "data/Placeholders.txt");
        sdi = SupplementalDataInfo.getInstance();
    }

    /**
     * Adapted from GenerateXMB.MapTransform
     *
     * @author jchye
     *
     */
    static class PlaceholderTransform implements Transform<String, Set<String>> {
        @Override
        public Set<String> transform(String source) {
            Set<String> placeholders = new LinkedHashSet<String>();
            String[] parts = source.split(";\\s+");
            for (String part : parts) {
                int equalsPos = part.indexOf('=');
                String placeholder = part.substring(0, equalsPos).trim();
                placeholders.add(placeholder);
            }
            return placeholders;
        }
    }

    @Override
    public CheckCLDR setCldrFileToCheck(CLDRFile cldrFile, Options options, List<CheckStatus> possibleErrors) {
        if (cldrFile == null) return this;
        skip = true;
        super.setCldrFileToCheck(cldrFile, options, possibleErrors);
        if (cldrFile.getLocaleID().equals("root")) {
            return this;
        }
        String locale = cldrFile.getLocaleID();
        col = Collator.getInstance(new ULocale(locale));
        spaceCol = Collator.getInstance(new ULocale(locale));
        spaceCol.setStrength(Collator.PRIMARY);

        CLDRFile resolvedFile = getResolvedCldrFileToCheck();
        boolean[] ok = new boolean[1];
        exemplars = safeGetExemplars("", possibleErrors, resolvedFile, ok);
        if (!ok[0]) exemplars = new UnicodeSet();

        if (exemplars == null) {
            CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType)
                .setSubtype(Subtype.noExemplarCharacters)
                .setMessage("No Exemplar Characters: {0}", new Object[] { this.getClass().getName() });
            possibleErrors.add(item);
            return this;
        }
        // UnicodeSet temp = resolvedFile.getExemplarSet("standard");
        // if (temp != null) exemplars.addAll(temp);
        UnicodeSet auxiliary = safeGetExemplars("auxiliary", possibleErrors, resolvedFile, ok); // resolvedFile.getExemplarSet("auxiliary",
        // CLDRFile.WinningChoice.WINNING);
        if (auxiliary != null) {
            exemplars.addAll(auxiliary);
        }

        if (CheckExemplars.USE_PUNCTUATION) {
            UnicodeSet punctuation = safeGetExemplars("punctuation", possibleErrors, resolvedFile, ok); // resolvedFile.getExemplarSet("auxiliary",
            if (punctuation != null) {
                exemplars.addAll(punctuation);
            }

            UnicodeSet numbers = getNumberSystemExemplars();
            exemplars.addAll(numbers);

            // TODO fix replacement character
            exemplars.add(STAND_IN);
        }

        exemplars.addAll(CheckExemplars.AlwaysOK).freeze();
        exemplarsPlusAscii = new UnicodeSet(exemplars).addAll(ASCII).freeze();

        skip = false;
        prettyPrint = new UnicodeSetPrettyPrinter()
            .setOrdering(col != null ? col : Collator.getInstance(ULocale.ROOT))
            .setSpaceComparator(col != null ? col : Collator.getInstance(ULocale.ROOT)
                .setStrength2(Collator.PRIMARY))
            .setCompressRanges(true);
        return this;
    }

    private UnicodeSet getNumberSystemExemplars() {
        String numberSystem = getCldrFileToCheck().getStringValue("//ldml/numbers/defaultNumberingSystem");
        String digits = sdi.getDigits(numberSystem);
        return new UnicodeSet().addAll(digits);
    }

    private UnicodeSet safeGetExemplars(String type, List<CheckStatus> possibleErrors, CLDRFile resolvedFile,
        boolean[] ok) {
        UnicodeSet result = null;
        try {
            result = resolvedFile.getExemplarSet(type, CLDRFile.WinningChoice.WINNING);
            ok[0] = true;
        } catch (IllegalArgumentException iae) {
            possibleErrors.add(new CheckStatus()
                .setCause(this).setMainType(CheckStatus.errorType).setSubtype(Subtype.couldNotAccessExemplars)
                .setMessage("Could not get exemplar set: " + iae.toString()));
            ok[0] = false;
        }
        return result;
    }

    public CheckCLDR handleCheck(String path, String fullPath, String value,
        Options options, List<CheckStatus> result) {
        if (fullPath == null) return this; // skip paths that we don't have
        if (value == null) return this; // skip values that we don't have ?
        if (skip) return this;
        if (path == null) {
            throw new InternalCldrException("Empty path!");
        } else if (getCldrFileToCheck() == null) {
            throw new InternalCldrException("no file to check!");
        }
        String sourceLocale = getResolvedCldrFileToCheck().getSourceLocaleID(path, otherPathStatus);

        // if we are an alias to another path, then skip
        // if (!path.equals(otherPathStatus.pathWhereFound)) {
        // return this;
        // }

        // now check locale source
        if (XMLSource.CODE_FALLBACK_ID.equals(sourceLocale)) {
            return this;
            // } else if ("root".equals(sourceLocale)) {
            // // skip eras for non-gregorian
            // if (true) return this;
            // if (path.indexOf("/calendar") >= 0 && path.indexOf("gregorian") <= 0) return this;
        }

        if (containsPart(path, EXEMPLAR_SKIPS)) {
            return this;
        }

        value = checkAndReplacePlaceholders(path, value, result);
        if (path.startsWith("//ldml/numbers/miscPatterns") && path.contains("[@type=\"range\"]")) {
            if (DISALLOWED_IN_RANGE.containsSome(value)) {
                result
                .add(new CheckStatus()
                    .setCause(this)
                    .setMainType(CheckStatus.errorType)
                    .setSubtype(Subtype.illegalCharactersInPattern)
                    .setMessage(
                        "Range patterns should not have letters.",
                        new Object[] {}));
            }
        }
        // Now handle date patterns.
        if (containsPart(path, DATE_PARTS)) {
            if (!extractDatePatternText(value, STAND_IN, justText)) {
                return this; // we are done, no text.
            }
            value = justText.toString();
            if (NUMBERS.containsSome(value)) {
                UnicodeSet disallowed = new UnicodeSet().addAll(value).retainAll(NUMBERS);
                addMissingMessage(disallowed, CheckStatus.errorType,
                    Subtype.patternCannotContainDigits,
                    Subtype.patternCannotContainDigits,
                    "cannot occur in date or time patterns", result);
            }
            if (path.endsWith("/hourFormat")) {
                UnicodeSet disallowed = new UnicodeSet().addAll(value)
                    .retainAll(DISALLOWED_HOUR_FORMAT);
                if (!disallowed.isEmpty()) {
                    addMissingMessage(disallowed, CheckStatus.errorType,
                        Subtype.patternContainsInvalidCharacters,
                        Subtype.patternContainsInvalidCharacters,
                        "cannot occur in the hour format", result);
                }
            }
        }

        if (path.startsWith("//ldml/posix/messages")) return this;

        UnicodeSet disallowed;

        if (path.contains("/currency") && path.contains("/symbol")) {
            if (null != (disallowed = containsAllCountingParens(exemplars, exemplarsPlusAscii, value))) {
                disallowed.removeAll(ALL_CURRENCY_SYMBOLS);
                disallowed.removeAll(LETTER); // Allow ASCII A-Z in currency symbols
                // String currency = new XPathParts().set(path).getAttributeValue(-2, "type");
                if (disallowed.size() > 0) {
                    // && asciiNotAllowed(getCldrFileToCheck().getLocaleID(), currency)) {
                    addMissingMessage(disallowed, CheckStatus.warningType,
                        Subtype.charactersNotInMainOrAuxiliaryExemplars,
                        Subtype.asciiCharactersNotInMainOrAuxiliaryExemplars, "are not in the exemplar characters",
                        result);
                }
            }
        } else if (path.contains("/gmtFormat") || path.contains("/gmtZeroFormat")) {
            if (null != (disallowed = containsAllCountingParens(exemplars, exemplarsPlusAscii, value))) {
                disallowed.removeAll(LETTER); // Allow ASCII A-Z in gmtFormat and gmtZeroFormat
                if (disallowed.size() > 0) {
                    addMissingMessage(disallowed, CheckStatus.warningType,
                        Subtype.charactersNotInMainOrAuxiliaryExemplars,
                        Subtype.asciiCharactersNotInMainOrAuxiliaryExemplars, "are not in the exemplar characters",
                        result);
                }
            }
        } else if (path.contains("/months") || path.contains("/quarters")) {
            if (null != (disallowed = containsAllCountingParens(exemplars, exemplarsPlusAscii, value))) {
                disallowed.removeAll("IVXivx"); // Allow Roman-numeral letters in month or quarter names
                if (path.contains("/calendar[@type=\"generic\"]/months")) {
                    disallowed.removeAll("M"); // Generic-calendar month names contain 'M' and do not get modified
                }
                if (disallowed.size() > 0) {
                    addMissingMessage(disallowed, CheckStatus.warningType,
                        Subtype.charactersNotInMainOrAuxiliaryExemplars,
                        Subtype.asciiCharactersNotInMainOrAuxiliaryExemplars, "are not in the exemplar characters",
                        result);
                }
            }
        } else if (path.contains("/localeDisplayNames") && !path.contains("/localeDisplayPattern")) {
            // test first for outside of the set.
            if (null != (disallowed = containsAllCountingParens(exemplars, exemplarsPlusAscii, value))) {
                if (path.contains("[@type=\"iso8601\"]")) {
                    disallowed.removeAll("ISO"); // Name of ISO8601 calendar may contain "ISO" regardless of native script
                }
                if (disallowed.size() > 0) {
                    addMissingMessage(disallowed, CheckStatus.warningType, Subtype.charactersNotInMainOrAuxiliaryExemplars,
                        Subtype.asciiCharactersNotInMainOrAuxiliaryExemplars, "are not in the exemplar characters", result);
                }
            }
            if (path.contains("/codePatterns")) {
                disallowed = new UnicodeSet().addAll(value).retainAll(NUMBERS);
                if (!disallowed.isEmpty()) {
                    addMissingMessage(disallowed, CheckStatus.errorType,
                        Subtype.patternCannotContainDigits,
                        Subtype.patternCannotContainDigits,
                        "cannot occur in locale fields", result);
                }
            }
        } else if (path.contains("/units")) {
            String noValidParentheses = IGNORE_PLACEHOLDER_PARENTHESES.matcher(value).replaceAll("");
            disallowed = new UnicodeSet().addAll(START_PAREN).addAll(END_PAREN)
                .retainAll(noValidParentheses);
            if (!disallowed.isEmpty()) {
                addMissingMessage(disallowed, CheckStatus.errorType,
                    Subtype.parenthesesNotAllowed,
                    Subtype.parenthesesNotAllowed,
                    "cannot occur in units", result);
            }
        } else {
            if (null != (disallowed = containsAllCountingParens(exemplars, exemplarsPlusAscii, value))) {
                addMissingMessage(disallowed, CheckStatus.warningType, Subtype.charactersNotInMainOrAuxiliaryExemplars,
                    Subtype.asciiCharactersNotInMainOrAuxiliaryExemplars, "are not in the exemplar characters", result);
            }
        }

        // check for spaces

        if (!value.equals(value.trim())) {
            if (!leadOrTrailWhitespaceOk.reset(path).find()) {
                result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType)
                    .setSubtype(Subtype.mustNotStartOrEndWithSpace)
                    .setMessage("This item must not start or end with whitespace, or be empty."));
            }
        }
        // if (value.contains("  ")) {
        // result.add(new
        // CheckStatus().setCause(this).setMainType(CheckStatus.errorType).setSubtype(Subtype.mustNotStartOrEndWithSpace)
        // .setMessage("This item must not contain two space characters in a row."));
        // }
        return this;
    }

    private String checkAndReplacePlaceholders(String path, String value, List<CheckStatus> result) {
        // add checks for patterns. Make sure that all and only the message format patterns have {n}
        Matcher matcher = patternMatcher.reset(value);
        Set<String> matchList = new HashSet<String>();
        StringBuffer placeholderBuffer = new StringBuffer();
        while (matcher.find()) {
            // Look for duplicate values.
            if (!matchList.add(matcher.group())) {
                placeholderBuffer.append(", ").append(matcher.group());
            }
        }
        Set<String> placeholders = null;
        PlaceholderStatus placeholderStatus = patternPlaceholders.getStatus(path);
        if (placeholderStatus != PlaceholderStatus.DISALLOWED) {
            placeholders = patternPlaceholders.get(path).keySet();
        }

        boolean supposedToHaveMessageFormatFields =
            // supposedToBeMessageFormat.reset(path).find()
            placeholders != null;

        if (supposedToHaveMessageFormatFields) {
            if (placeholderBuffer.length() > 0) {
                if (placeholderStatus != PlaceholderStatus.MULTIPLE) {
                    result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType)
                        .setSubtype(Subtype.extraPlaceholders)
                        .setMessage("Remove duplicates of{0}",
                            new Object[] { placeholderBuffer.substring(1) }));
                }
            }
            placeholderBuffer.setLength(0);
            // Check that the needed placeholders are there.
            if (placeholders == null) placeholders = new HashSet<String>();
            for (String placeholder : placeholders) {
                if (!matchList.contains(placeholder)) {
                    placeholderBuffer.append(", ").append(placeholder);
                }
            }
            boolean placeholdersMissing = false;
            if (placeholderBuffer.length() > 0) {
                // Check
                if (placeholderStatus == PlaceholderStatus.LOCALE_DEPENDENT && (path.contains("[@count=") || path.contains("[@ordinal="))) {
                    PluralRules rules = PluralRules.forLocale(new ULocale(getCldrFileToCheck().getLocaleID()));
                    XPathParts parts = XPathParts.getFrozenInstance(path);
                    String keyword = parts.getAttributeValue(-1, "count");
                    if (keyword == null) {
                        keyword = parts.getAttributeValue(-1, "ordinal");
                    }
                    placeholdersMissing = rules.getUniqueKeywordValue(keyword) == PluralRules.NO_UNIQUE_VALUE;
                } else {
                    placeholdersMissing = placeholderStatus == PlaceholderStatus.REQUIRED;
                }
            }
            if (placeholdersMissing) {
                result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType)
                    .setSubtype(Subtype.missingPlaceholders)
                    .setMessage("This message pattern is missing placeholder(s){0}. See the English for an example.",
                        new Object[] { placeholderBuffer.substring(1) }));
            }
            // Check for extra placeholders.
            matchList.removeAll(placeholders);
            if (matchList.size() > 0) {
                String list = matchList.toString();
                list = list.substring(1, list.length() - 1);
                result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType)
                    .setSubtype(Subtype.extraPlaceholders)
                    .setMessage("Extra placeholders {0} should be removed.",
                        new Object[] { list }));
            }
            // check the other characters in the message format patterns
            value = patternMatcher.replaceAll(STAND_IN);
        } else if (matchList.size() > 0 && placeholderStatus == PlaceholderStatus.DISALLOWED) { // non-message field has
            // placeholder values
            result.add(new CheckStatus()
                .setCause(this)
                .setMainType(CheckStatus.errorType)
                .setSubtype(Subtype.shouldntHavePlaceholders)
                .setMessage(
                    "This field is not a message pattern, and should not have '{0}, {1},' etc. See the English for an example.",
                    new Object[] {}));
            // end checks for patterns
        }
        return value;
    }

    /**
     * Checks if ASCII characters are allowed in a currency symbol in the specified locale.
     * @param localeID the locale ID that the currency is in
     * @param currency the currency to be checked
     * @return true if ASCII is not allowed
     */
    private boolean asciiNotAllowed(String localeID, String currency) {
        // Don't allow ascii at all for bidi scripts.
        String charOrientation = getResolvedCldrFileToCheck().getStringValue(
            "//ldml/layout/orientation/characterOrder");
        if (charOrientation.equals("right-to-left")) {
            return true;
        }

        // Get script of locale. if Latn, quit.
        LocaleIDParser parser = new LocaleIDParser().set(localeID);
        String script = parser.getScript();
        if (script.length() == 0) {
            localeID = sdi.getLikelySubtags().get(localeID);
            if (localeID == null) {
                localeID = sdi.getLikelySubtags().get(parser.getLanguage());
                if (localeID == null) {
                    throw new IllegalArgumentException(
                        "A likely subtag for " + parser.getLanguage() +
                        " is required to get its script.");
                }
            }
            script = parser.set(localeID).getScript();
        }
        if (script.equals("Latn")) {
            return false;
        }

        // Enforce checking of for other non-Latin scripts, for all currencies
        // whose countries use that script, e.g. Russian should have Cyrillic
        // currency symbols for modern currencies of countries with official
        // languages whose script is Cyrillic (Bulgaria, Serbia, ...).
        Set<String> currencies = getCurrenciesForScript(script);
        return currencies != null && currencies.contains(currency);
    }

    private Set<String> getCurrenciesForScript(String script) {
        if (scriptToCurrencies != null) return scriptToCurrencies.get(script);

        // Get mapping of scripts to the territories that use that script in
        // any of their primary languages.
        Relation scriptToTerritories = new Relation(new HashMap<String, Set<String>>(), HashSet.class);
        for (String lang : sdi.getBasicLanguageDataLanguages()) {
            BasicLanguageData langData = sdi.getBasicLanguageDataMap(lang).get(Type.primary);
            if (langData == null) {
                continue;
            }
            for (String curScript : langData.getScripts()) {
                scriptToTerritories.putAll(curScript, langData.getTerritories());
            }
        }

        // For each territory, get all of its legal tender currencies.
        Date now = new Date(System.currentTimeMillis());
        scriptToCurrencies = new Relation(new HashMap<String, Set<String>>(), HashSet.class);
        for (Object curScript : scriptToTerritories.keySet()) {
            Set<String> territories = scriptToTerritories.get(curScript);
            Set<String> currencies = new HashSet<String>();
            for (String territory : territories) {
                Set<CurrencyDateInfo> currencyInfo = sdi.getCurrencyDateInfo(territory);
                for (CurrencyDateInfo info : currencyInfo) {
                    if (info.isLegalTender() && info.getEnd().compareTo(now) > 0) {
                        currencies.add(info.getCurrency());
                    }
                }
            }
            scriptToCurrencies.putAll(curScript, currencies);
        }
        return scriptToCurrencies.get(script);
    }

    /**
     * Extracts just the text from a date field, replacing all the variable fields by variableReplacement. Return null
     * if
     * there is an error (a different test will find that error).
     */
    public boolean extractDatePatternText(String value, String variableReplacement, StringBuilder justText) {
        boolean haveText = false;
        try {
            formatParser.set(value);
        } catch (Exception e) {
            return false; // give up, it is illegal
        }
        boolean doReplacement = variableReplacement != null && variableReplacement.length() > 0;
        justText.setLength(0);
        for (Object item : formatParser.getItems()) {
            if (item instanceof String) {
                justText.append(item);
                haveText = true;
            } else {
                if (doReplacement) {
                    justText.append(variableReplacement);
                }
            }
        }
        return haveText;
    }

    public boolean containsPart(String source, String... segments) {
        for (int i = 0; i < segments.length; ++i) {
            if (source.indexOf(segments[i]) > 0) {
                return true;
            }
        }
        return false;
    }

    static final String TEST = "؉";

    private void addMissingMessage(UnicodeSet missing, CheckStatus.Type warningVsError, Subtype subtype,
        Subtype subtypeAscii,
        String qualifier, List<CheckStatus> result) {
        String fixedMissing = prettyPrint.format(missing);
        BitSet scripts = new BitSet();
        for (String s : missing) {
            final int script = UScript.getScript(s.codePointAt(0));
            if (script == UScript.INHERITED || script == UScript.COMMON) {
                continue;
            }
            scripts.set(script);
        }
        StringBuilder scriptString = new StringBuilder();
        if (!scripts.isEmpty()) {
            scriptString.append("{");
            for (int i = scripts.nextSetBit(0); i >= 0; i = scripts.nextSetBit(i + 1)) {
                if (scriptString.length() > 1) {
                    scriptString.append(", ");
                }
                scriptString.append(UScript.getName(i));
            }
            scriptString.append("}");
        }
        result
        .add(new CheckStatus()
            .setCause(this)
            .setMainType(warningVsError)
            .setSubtype(ASCII.containsAll(missing) ? subtypeAscii : subtype)
            .setMessage(
                "The characters \u200E{0}\u200E {1} {2}. "
                    +
                    "For what to do, see <i>Handling Warnings</i> in <a target='CLDR-ST-DOCS' href='http://cldr.org/translation/characters#TOC-Handing-Warnings'>Characters</a>.",
                    new Object[] { fixedMissing, scriptString, qualifier }));
    }

    static final Normalizer2 NFC = Normalizer2.getInstance(null, "nfc", Normalizer2.Mode.COMPOSE);

    /**
     * Return null if ok, otherwise UnicodeSet of bad characters
     *
     * @param exemplarSet
     * @param value
     * @return
     */
    private UnicodeSet containsAllCountingParens(UnicodeSet exemplarSet, UnicodeSet exemplarSetPlusASCII, String value) {
        UnicodeSet result = null;
        if (exemplarSet.containsAll(value)) {
            return result;
        }

        // Normalize
        value = NFC.normalize(value);

        // if we failed, then check that everything outside of () is ok.
        // and everything inside parens is either ASCII or in the set
        int lastPos = 0;
        while (true) {
            int start = START_PAREN.findIn(value, lastPos, false);
            String outside = value.substring(lastPos, start);
            result = addDisallowedItems(exemplarSet, outside, result);
            if (start == value.length()) {
                break; // all done
            }
            ++start;
            int end = END_PAREN.findIn(value, start, false);
            // don't worry about mixed brackets
            String inside = value.substring(start, end);
            result = addDisallowedItems(exemplarSetPlusASCII, inside, result);
            if (end == value.length()) {
                break; // all done
            }
            lastPos = end + 1;
        }
        return result;
    }

    private UnicodeSet addDisallowedItems(UnicodeSet exemplarSet, String outside, UnicodeSet result) {
        if (!exemplarSet.containsAll(outside)) {
            if (result == null) {
                result = new UnicodeSet();
            }
            result.addAll(new UnicodeSet().addAll(outside).removeAll(exemplarSet));
        }
        return result;
    }
}
