"""DeciMethod Research Statistics Lab: fixed, bounded SciPy analyses.

No uploaded code executes. The original CSV, declared analysis and row lineage
remain in each result. Run locally: python engine.py project.json > result.json
"""
from datetime import datetime, timezone
from decimal import Decimal, InvalidOperation
import json
import math
import re
import sys
import warnings as python_warnings

import numpy as np
import scipy
from scipy import stats

METHOD_VERSION = 'research-statistics/1.0.0'
LIMITS = {'bytes': 2097152, 'rows': 5000, 'columns': 40, 'cells': 100000,
          'cellChars': 4000, 'maxAbsNumeric': 1e12, 'maxPlannedSampleSize': 1000000000}
ANALYSIS_DESIGNS = {'descriptive': 'descriptive', 'welch': 'independent',
                    'paired': 'paired', 'pearson': 'observational', 'precision': 'planning'}
NUMBER = re.compile(r'^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$', re.ASCII)
CONTROL = re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]')
JS_WHITESPACE = '\u0009\u000a\u000b\u000c\u000d\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff'


def _require(condition, message):
    if not condition:
        raise ValueError(message)


def _length(text):
    return len(text.encode('utf-16-le', errors='surrogatepass')) // 2


def _trim(text):
    return text.strip(JS_WHITESPACE)


def _text(value, label, maximum=4000, required=False):
    _require(isinstance(value, str) and _length(value) <= maximum and not CONTROL.search(value),
             f'{label} must be text of at most {maximum} characters without unsupported control characters.')
    _require(not required or bool(_trim(value)), f'{label} must not be blank.')
    return value


def _object(value, label, names):
    _require(isinstance(value, dict), f'{label} must be an object.')
    _require(all(key in names for key in value), f'{label} contains an unrecognized setting.')
    return value


def _finite_number(value, label):
    _require(not isinstance(value, bool) and isinstance(value, (int, float)) and math.isfinite(value),
             f'{label} must be a finite number.')
    return value


def parse_csv(text):
    """Strict comma CSV matching browser parsing, retaining every data record."""
    _require(isinstance(text, str) and len(text.encode('utf-8')) <= LIMITS['bytes'], 'CSV must be text of at most 2 MiB.')
    _require(not CONTROL.search(text), 'CSV contains unsupported control characters.')
    source = text[1:] if text.startswith('\ufeff') else text
    records, row, characters = [], [], []
    quoted, closed, beginning = False, False, True

    def cell():
        nonlocal characters, closed, beginning
        value = ''.join(characters)
        _require(_length(value) <= LIMITS['cellChars'], 'CSV cells must contain at most 4000 characters.')
        row.append(value)
        characters, closed, beginning = [], False, True
        _require(len(row) <= LIMITS['columns'], 'CSV supports at most 40 columns.')

    def record():
        nonlocal row
        cell()
        records.append(row)
        row = []
        _require(len(records) <= LIMITS['rows'] + 1, 'CSV supports at most 5000 data records.')

    index = 0
    while index < len(source):
        character = source[index]
        if quoted:
            if character == '"':
                if index + 1 < len(source) and source[index + 1] == '"':
                    characters.append('"')
                    index += 1
                else:
                    quoted, closed = False, True
            else:
                characters.append(character)
        elif character == ',':
            cell()
        elif character in '\r\n':
            if character == '\r' and index + 1 < len(source) and source[index + 1] == '\n':
                index += 1
            record()
        elif closed:
            raise ValueError('Unexpected text after a quoted CSV field.')
        elif character == '"':
            _require(beginning, 'A CSV quote must begin a field or escape another quote.')
            quoted, beginning = True, False
        else:
            characters.append(character)
            beginning = False
        _require(len(characters) <= LIMITS['cellChars'], 'CSV cells must contain at most 4000 characters.')
        index += 1
    _require(not quoted, 'CSV has an unterminated quoted field.')
    if characters or row or closed or not beginning:
        record()
    _require(len(records) >= 2, 'CSV needs a header and at least one data record.')
    headers = [_trim(value) for value in records.pop(0)]
    _require(all(headers) and len(set(headers)) == len(headers), 'CSV headers must be nonempty and unique after trimming.')
    _require(len(records) * len(headers) <= LIMITS['cells'], 'CSV supports at most 100000 data cells.')
    for position, values in enumerate(records, 1):
        _require(len(values) == len(headers), f'CSV data record {position} has {len(values)} fields; the header has {len(headers)}.')
    return {'headers': headers, 'rows': records}


def validate_project(raw):
    _object(raw, 'Project', ['title', 'source', 'question', 'design', 'analysis', 'phase', 'csv', 'columns', 'groups', 'confidence', 'missingTokens', 'planning'])
    title = _trim(_text(raw.get('title'), 'Project title', 240, True))
    source = _text(raw.get('source', ''), 'Source and version')
    question = _text(raw.get('question', ''), 'Study question and measurement context')
    analysis = raw.get('analysis', 'descriptive')
    _require(isinstance(analysis, str) and analysis in ANALYSIS_DESIGNS, 'Choose a supported statistics workflow.')
    design = raw.get('design', ANALYSIS_DESIGNS[analysis])
    _require(design == ANALYSIS_DESIGNS[analysis], f'The {analysis} analysis requires the {ANALYSIS_DESIGNS[analysis]} study design.')
    phase = raw.get('phase', 'exploratory')
    _require(phase in ('exploratory', 'prespecified'), 'Analysis status must be exploratory or prespecified.')
    confidence = _finite_number(raw.get('confidence', .95), 'Confidence level')
    _require(.8 <= confidence <= .999, 'Confidence level must lie between 0.8 and 0.999.')
    csv = _text(raw.get('csv', ''), 'CSV text', LIMITS['bytes'])
    _require(len(csv.encode('utf-8')) <= LIMITS['bytes'], 'CSV must be at most 2 MiB.')
    given_columns = _object(raw.get('columns', {}), 'Column selection', ['x', 'y', 'group'])
    columns = {name: _text(given_columns.get(name, ''), f'Selected {name} column') for name in ('x', 'y', 'group')}
    given_groups = _object(raw.get('groups', {}), 'Group labels', ['a', 'b'])
    groups = {name: _text(given_groups.get(name, ''), f'Group {name} label') for name in ('a', 'b')}
    tokens = raw.get('missingTokens', [])
    _require(isinstance(tokens, list) and len(tokens) <= 30, 'Declare at most 30 missing-value tokens.')
    missing_tokens = [_text(value, 'Missing token', 200, True) for value in tokens]
    _require(len(set(missing_tokens)) == len(missing_tokens) and all(value == _trim(value) for value in missing_tokens), 'Missing tokens must be unique exact values after trimming. Blank cells are already missing.')
    given_plan = _object(raw.get('planning', {}), 'Precision plan', ['estimand', 'margin', 'sd', 'proportion'])
    planning = {'estimand': 'mean', 'margin': 1, 'sd': 5, 'proportion': .5}
    if analysis == 'precision':
        planning['estimand'] = given_plan.get('estimand', 'mean')
        _require(planning['estimand'] in ('mean', 'proportion'), 'Precision planning supports a mean or a proportion.')
        planning['margin'] = _finite_number(given_plan.get('margin', 1), 'Planning margin')
        _require(0 < planning['margin'] <= 1e12, 'Planning requires a positive margin of at most 1e12.')
        if planning['estimand'] == 'mean':
            planning['sd'] = _finite_number(given_plan.get('sd', 5), 'Planning standard deviation')
            _require(0 < planning['sd'] <= 1e12, 'The planning standard deviation must be positive and at most 1e12.')
        else:
            planning['proportion'] = _finite_number(given_plan.get('proportion', .5), 'Planning proportion')
            _require(0 < planning['proportion'] < 1, 'The planning proportion must lie strictly between 0 and 1.')
            _require(planning['margin'] < 1, 'A proportion margin must be smaller than 1.')
    if analysis != 'precision':
        headers = parse_csv(csv)['headers']
        needed = ['x', 'group'] if analysis == 'welch' else ['x', 'y'] if analysis in ('paired', 'pearson') else ['x']
        for key in needed:
            _require(columns[key] in headers, f'Select an existing {key} column from the CSV.')
        _require(len(set(columns[key] for key in needed)) == len(needed), 'Select different columns for the requested roles.')
        if analysis == 'welch':
            _require(groups['a'] and groups['b'] and groups['a'] != groups['b'] and
                     all(value == _trim(value) and value not in missing_tokens for value in groups.values()),
                     'Choose two distinct nonmissing group labels, matched exactly after trimming.')
    return {'title': title, 'source': source, 'question': question, 'design': design,
            'analysis': analysis, 'phase': phase, 'csv': csv, 'columns': columns,
            'groups': groups, 'confidence': confidence, 'missingTokens': missing_tokens, 'planning': planning}


def _numeric(value, column, source_row, missing_tokens):
    value = _trim(value)
    if value == '' or value in missing_tokens:
        return None
    message = f'Data record {source_row}, column {column}: supply a finite decimal number within ±1e12, or declare its exact missing token.'
    _require(bool(NUMBER.fullmatch(value)), message)
    try:
        exact = Decimal(value)
        numeric = float(value)
    except (ValueError, OverflowError, InvalidOperation) as error:
        raise ValueError(message) from error
    _require(math.isfinite(numeric) and abs(numeric) <= LIMITS['maxAbsNumeric'] and not (numeric == 0 and exact != 0), message)
    _require(not (exact == exact.to_integral_value() and abs(exact) > 9007199254740991), message)
    return numeric


def _interval(low, high, confidence):
    return {'low': float(low), 'high': float(high), 'confidence': confidence}


def _sample_sd(array):
    scale = float(np.max(np.abs(array))) if len(array) else 0
    if scale == 0:
        return 0.0
    result = float(np.std(array / scale, ddof=1)) * scale
    _require(not (result == 0 and np.ptp(array) != 0), 'Numerical observations have unresolved variation at this scale. Rescale the selected units before analysis.')
    return result


def _sample_mean(array):
    if not len(array):
        return None
    total = math.fsum(float(value) for value in array)
    mean = total / len(array)
    _require(not (mean == 0 and total != 0), 'The sample mean is unresolved at this numerical scale. Rescale the selected units before analysis.')
    return mean


def _description(values, label, column, confidence):
    count = len(values)
    result = {'label': label, 'column': column, 'n': count, 'mean': None, 'sd': None,
              'min': None, 'q1': None, 'median': None, 'q3': None, 'max': None,
              'meanCI': None, 'meanCIReason': 'At least two observations and nonzero sample variance are required for the t-based mean interval.'}
    if not count:
        return result
    array = np.asarray(values, dtype=float)
    q1, median, q3 = np.quantile(array, [.25, .5, .75], method='linear')
    result.update(mean=_sample_mean(array), min=float(np.min(array)), q1=float(q1), median=float(median), q3=float(q3), max=float(np.max(array)))
    if count >= 2:
        sd = _sample_sd(array)
        result['sd'] = sd
        if sd > 0:
            half = float(stats.t.ppf((1 + confidence) / 2, count - 1)) * sd / math.sqrt(count)
            result['meanCI'] = _interval(result['mean'] - half, result['mean'] + half, confidence)
            result['meanCIReason'] = None
    return result


def _inference(method, estimand, estimate=None, reason=None):
    return {'method': method, 'estimand': estimand, 'estimate': estimate, 'statistic': None,
            'pValue': None, 'df': None, 'confidenceInterval': None, 'effectSize': None,
            'unavailableReason': reason}


def _clean_json(value):
    if isinstance(value, dict):
        return {str(key): _clean_json(item) for key, item in value.items()}
    if isinstance(value, (list, tuple, np.ndarray)):
        return [_clean_json(item) for item in value]
    if isinstance(value, (np.integer,)):
        return int(value)
    if isinstance(value, (float, np.floating)):
        return float(value) if math.isfinite(float(value)) else None
    if isinstance(value, np.bool_):
        return bool(value)
    return value


def _plan(project):
    plan, confidence = project['planning'], project['confidence']
    z = float(stats.norm.ppf((1 + confidence) / 2))
    try:
        raw = (z * plan['sd'] / plan['margin']) ** 2 if plan['estimand'] == 'mean' else z ** 2 * plan['proportion'] * (1 - plan['proportion']) / plan['margin'] ** 2
    except (OverflowError, ZeroDivisionError) as error:
        raise ValueError('The requested precision exceeds this planner’s numeric range. Increase the margin or rescale the outcome.') from error
    _require(math.isfinite(raw) and raw <= LIMITS['maxPlannedSampleSize'], 'The requested precision requires more than 1 billion observations or exceeds numeric range. Increase the margin or rescale the outcome.')
    count = max(1, math.ceil(raw))
    return {'estimand': plan['estimand'], 'method': 'Normal-approximation precision planning; no finite-population correction',
            'confidence': confidence, 'z': z, 'margin': plan['margin'],
            'sd': plan['sd'] if plan['estimand'] == 'mean' else None,
            'proportion': plan['proportion'] if plan['estimand'] == 'proportion' else None,
            'rawSampleSize': raw, 'requiredSampleSize': count,
            'assumptions': ['Independent simple-random observations; complete usable records, before allowance for nonresponse or clustering.',
                            'The mean formula treats the planning standard deviation as known. Estimated SD and small-sample t planning require separate evaluation.',
                            'The proportion formula uses a normal approximation; p=0.5 is conservative for a fixed absolute margin. Rare proportions and small expected counts require separate methods.',
                            'This is precision planning for an interval, not statistical power, a trial design, or a guarantee of representativeness.']}


def run_analysis(raw):
    project = validate_project(raw)
    notices = []
    if not _trim(project['source']):
        notices.append('No source or dataset version has been documented.')
    if not _trim(project['question']):
        notices.append('No study question, row meaning or measurement units have been documented.')
    notices.append('Exploratory analyses are hypothesis-generating; repeated analyses and outcome selection can inflate false-positive findings.' if project['phase'] == 'exploratory' else 'Prespecified is a user declaration; the lab has not verified a preregistered protocol or the timing of this choice.')
    assumptions = [
        'The selected design and column roles are declared by the user. The lab does not verify sampling, independence, pairing, measurement scales or a causal interpretation.',
        'CSV field values are trimmed before interpretation. Blank cells and exact case-sensitive declared missing tokens are missing; no values are imputed.',
        'Only selected numerical roles in the scoped analysis are parsed. A nonmissing invalid numerical value causes an error, rather than silently being excluded.',
        'Source row numbers identify one-based CSV data records, not physical text lines. Complete-case exclusions and selected-group exclusions remain in the row ledger.',
        'Reported tests are two-sided. Mean intervals use Student t; Welch comparisons allow unequal variances; paired analyses use within-row x−y differences.',
        'Normal-population or adequate large-sample approximations and independent observations remain contextual assumptions. The lab does not automatically choose tests from a normality pretest.',
        'Pearson inference uses its exact null distribution and a Fisher-transform interval with at least four complete pairs in this release. Pearson measures linear association, not causation.',
        'Quartiles use linear interpolation at (n−1)p (type 7). Standard deviations use the sample denominator n−1.',
        'A confidence level describes the long-run coverage of the interval procedure under its assumptions. A p-value is not the probability that a null hypothesis is true.',
        'The mean difference in the declared outcome units is the primary comparison effect estimate; a small p-value alone does not establish practical importance.',
        'All inference is unadjusted for multiple testing. The selected analysis and phase remain in the project record; there is no automatic accept/reject badge.',
        'Calculations use finite double-precision arithmetic, bounded numerical observations and the recorded scientific-library versions. A completed run has not been independently reviewed.'
    ]
    rows = {'total': 0, 'used': 0, 'excluded': 0, 'usedSourceRows': [], 'excludedRows': []}
    descriptions, inference, planning = [], None, None
    with python_warnings.catch_warnings(record=True) as recorded:
        python_warnings.simplefilter('always')
        if project['analysis'] == 'precision':
            planning = _plan(project)
            notices.append('The sample-size result is an approximate precision plan, not a power analysis. Nonresponse, clustering and study-specific constraints are not included.')
        else:
            table = parse_csv(project['csv'])
            headers, records = table['headers'], table['rows']
            positions = {name: headers.index(value) for name, value in project['columns'].items() if value in headers}
            xs, ys, a_values, b_values = [], [], [], []
            for source_row, record in enumerate(records, 1):
                if project['analysis'] == 'welch':
                    group = _trim(record[positions['group']])
                    if group == '' or group in project['missingTokens']:
                        rows['excludedRows'].append({'sourceRow': source_row, 'reason': 'Missing selected group label.'})
                        continue
                    if group not in project['groups'].values():
                        rows['excludedRows'].append({'sourceRow': source_row, 'reason': 'Group label is outside the two declared comparison groups.'})
                        continue
                x = _numeric(record[positions['x']], project['columns']['x'], source_row, project['missingTokens'])
                y = _numeric(record[positions['y']], project['columns']['y'], source_row, project['missingTokens']) if project['analysis'] in ('paired', 'pearson') else None
                if x is None or (project['analysis'] in ('paired', 'pearson') and y is None):
                    rows['excludedRows'].append({'sourceRow': source_row, 'reason': 'Missing a selected numerical value required for this analysis.'})
                    continue
                rows['usedSourceRows'].append(source_row)
                xs.append(x)
                if project['analysis'] in ('paired', 'pearson'):
                    ys.append(y)
                if project['analysis'] == 'welch':
                    (a_values if group == project['groups']['a'] else b_values).append(x)
            rows.update(total=len(records), used=len(rows['usedSourceRows']), excluded=len(rows['excludedRows']))
            if rows['excluded']:
                notices.append(f'{rows["excluded"]} source record(s) were excluded under the documented selected-group or complete-case rules. The analysis population can differ from the full extract.')
            confidence = project['confidence']
            if project['analysis'] == 'descriptive':
                descriptions = [_description(xs, project['columns']['x'], project['columns']['x'], confidence)]
                if not xs:
                    notices.append('No complete numerical observations remain for the selected column.')
            elif project['analysis'] == 'welch':
                a, b = np.asarray(a_values), np.asarray(b_values)
                descriptions = [_description(a, f'Group {project["groups"]["a"]}', project['columns']['x'], confidence),
                                _description(b, f'Group {project["groups"]["b"]}', project['columns']['x'], confidence)]
                difference = _sample_mean(a) - _sample_mean(b) if len(a) and len(b) else None
                inference = _inference('Welch independent two-sample t-test', 'Mean of group A − mean of group B', difference)
                inference.update(groupA=project['groups']['a'], groupB=project['groups']['b'], nA=len(a), nB=len(b))
                if len(a) < 2 or len(b) < 2:
                    inference['unavailableReason'] = 'Each selected group needs at least two complete observations.'
                elif np.ptp(a) == 0 and np.ptp(b) == 0:
                    inference['unavailableReason'] = 'Both groups have zero sample variance. The observed difference is retained, but t inference and its interval are unavailable.'
                else:
                    scale = float(max(np.max(np.abs(a)), np.max(np.abs(b))))
                    test = stats.ttest_ind(a / scale, b / scale, equal_var=False, alternative='two-sided')
                    ci = test.confidence_interval(confidence_level=confidence)
                    inference.update(statistic=float(test.statistic), pValue=float(test.pvalue), df=float(test.df), confidenceInterval=_interval(ci.low * scale, ci.high * scale, confidence))
            elif project['analysis'] == 'paired':
                a, b = np.asarray(xs), np.asarray(ys)
                differences = a - b
                descriptions = [_description(a, project['columns']['x'], project['columns']['x'], confidence),
                                _description(b, project['columns']['y'], project['columns']['y'], confidence),
                                _description(differences, 'Within-row x − y differences', f'{project["columns"]["x"]} − {project["columns"]["y"]}', confidence)]
                inference = _inference('Paired t-test', 'Mean within-row x − y difference', _sample_mean(differences) if len(a) else None)
                inference['nPairs'] = len(a)
                if len(a) < 2:
                    inference['unavailableReason'] = 'At least two complete row-matched pairs are required.'
                elif np.ptp(differences) == 0:
                    inference['unavailableReason'] = 'Within-row differences have zero sample variance. The observed difference is retained, but paired t inference and its interval are unavailable.'
                else:
                    scale = float(np.max(np.abs(differences)))
                    test = stats.ttest_1samp(differences / scale, 0, alternative='two-sided')
                    ci = test.confidence_interval(confidence_level=confidence)
                    inference.update(statistic=float(test.statistic), pValue=float(test.pvalue), df=float(test.df), confidenceInterval=_interval(ci.low * scale, ci.high * scale, confidence))
            elif project['analysis'] == 'pearson':
                a, b = np.asarray(xs), np.asarray(ys)
                descriptions = [_description(a, project['columns']['x'], project['columns']['x'], confidence), _description(b, project['columns']['y'], project['columns']['y'], confidence)]
                inference = _inference('Pearson linear correlation', 'Pearson population correlation coefficient')
                inference['nPairs'] = len(a)
                if len(a) < 2:
                    inference['unavailableReason'] = 'At least two complete observations are required to calculate a correlation.'
                elif np.ptp(a) == 0 or np.ptp(b) == 0:
                    inference['unavailableReason'] = 'A selected variable is constant. A correlation coefficient and its inference are undefined.'
                else:
                    test = stats.pearsonr(a / np.max(np.abs(a)), b / np.max(np.abs(b)), alternative='two-sided')
                    inference['estimate'] = float(test.statistic)
                    inference['effectSize'] = {'name': 'Pearson r', 'value': float(test.statistic)}
                    if len(a) < 4:
                        inference['unavailableReason'] = 'The descriptive correlation is retained. At least four complete independent pairs are required for inference and a Fisher interval in this release.'
                    else:
                        ci = test.confidence_interval(confidence_level=confidence)
                        inference.update(statistic=float(test.statistic), pValue=float(test.pvalue), confidenceInterval=_interval(ci.low, ci.high, confidence))
            unresolved = inference and (any(not math.isfinite(inference[key]) for key in ('statistic', 'pValue', 'df') if inference[key] is not None) or
                                        (inference['confidenceInterval'] is not None and any(not math.isfinite(inference['confidenceInterval'][key]) for key in ('low', 'high'))))
            if inference and not inference['unavailableReason'] and unresolved:
                inference.update(statistic=None, pValue=None, df=None, confidenceInterval=None, unavailableReason='The inferential calculation is numerically unresolved. Rescale the selected units or obtain specialist review.')
            if inference and inference['unavailableReason']:
                notices.append(inference['unavailableReason'])
    if recorded:
        categories = sorted({item.category.__name__ for item in recorded})
        notices.append('Scientific-library diagnostic warning (' + ', '.join(categories) + '): numerical precision or distributional degeneracy needs review. Inspect variable scale, resolution and the declared design before interpreting inference.')
    result = {'methodVersion': METHOD_VERSION, 'generatedAt': datetime.now(timezone.utc).isoformat(),
              'project': project, 'versions': {'python': sys.version.split()[0], 'numpy': np.__version__, 'scipy': scipy.__version__, 'pyodide': '0.28.3' if sys.platform == 'emscripten' else None},
              'analysis': project['analysis'], 'descriptives': descriptions, 'inference': inference,
              'planning': planning, 'rows': rows, 'warnings': notices, 'assumptions': assumptions,
              'reviewStatus': 'Not independently reviewed'}
    return _clean_json(result)


if __name__ == '__main__' and sys.platform != 'emscripten':
    import argparse
    parser = argparse.ArgumentParser(description='Run a fixed DeciMethod statistics project locally.')
    parser.add_argument('project', help='Path to project JSON exported by the lab.')
    arguments = parser.parse_args()
    with open(arguments.project, encoding='utf-8') as project_file:
        _project_input = json.load(project_file)
    print(json.dumps(run_analysis(_project_input), allow_nan=False, ensure_ascii=False, indent=2))
