#!/usr/bin/env python3
"""Recalculate the September 5, 2026 witness-statistics editorial updates.

Usage:
    python3 tools/verify_witness_stats.py --data-root /path/to/datasets

Place each CSV beneath DATA_ROOT/<repository>/<relative CSV path>. Files
ending in .csv.data are also accepted, to distinguish downloaded LFS data
from .csv pointer files. This script does not download files or execute dataset code.
It checks every file against the byte length and SHA-256 in the pinned
Git LFS pointers below, then writes the calculations as JSON to stdout.

Sources are public repositories owned by mandrigin on GitHub. Each entry
records the source commit, file path, byte length, and LFS SHA-256. The
resolver's quantile-analysis.py divides bytes by 1024**2 and supports a
lower bound only. Its README's reported mean uses rows from block 3M
through the file's actual end, despite describing an upper bound of 6M.
The KV percentile.py uses BlockWitnessSize with adjust_keys=False.

Only Python's standard library is required. Parsing the four files uses
several hundred MB of memory. Original plotted code is not executed.
"""

import argparse
from array import array
import csv
import hashlib
import json
from pathlib import Path


MIB = 1024 ** 2
KV_FIRST, KV_LAST = 5_000_000, 8_000_000
SOURCES = {
    'resolver': {
        'repository': 'ethereum-mainnet-resolver-witness-stats',
        'commit': '9accc939c223831ba505367e4f6ac6bdccfaabf8',
        'path': 'cache_1_000_000/semi_stateless_witnesses.db.stats.1.csv',
        'bytes': 54_029_765,
        'sha256': 'de5fd0baad058d457b6a1271b672ee0195b535fc8e584babedc75511a166e040',
    },
    'opcode': {
        'repository': 'ethereum-mainnet-kv-witness-data',
        'commit': 'ab170bf6da6719bbc441cd30a92c990cbbc455d7',
        'path': 'stats_opcode.csv',
        'bytes': 187_215_204,
        'sha256': 'da44328ed4e9291ab2daab1c867572bbc2713fbf5cccbc7d12569d1e20ebde83',
    },
    'kv_compressed': {
        'repository': 'ethereum-mainnet-kv-witness-data',
        'commit': 'ab170bf6da6719bbc441cd30a92c990cbbc455d7',
        'path': 'stats_kv_compressed.csv',
        'bytes': 144_912_480,
        'sha256': '67ee0fed24b3ad983abf5b2e640dddfda8e0e81409c2329473f312a106854924',
    },
    'kv_uncompressed': {
        'repository': 'ethereum-mainnet-kv-witness-data',
        'commit': 'ab170bf6da6719bbc441cd30a92c990cbbc455d7',
        'path': 'stats_kv_uncompressed.csv',
        'bytes': 149_274_767,
        'sha256': 'a969fa7db5e0373d73c161b2686c94dbf30f576a8822370eb8b139319c730e65',
    },
}


def checked_files(data_root):
    files = {}
    for name, source in SOURCES.items():
        path = data_root / source['repository'] / source['path']
        if path.with_suffix('.csv.data').is_file():
            path = path.with_suffix('.csv.data')
        if path.stat().st_size != source['bytes']:
            raise ValueError(f'{path}: incorrect file size; use CSV data, not an LFS pointer')
        with path.open('rb') as stream:
            digest = hashlib.file_digest(stream, 'sha256').hexdigest()
        if digest != source['sha256']:
            raise ValueError(f'{path}: SHA-256 does not match the pinned source')
        files[name] = path
    return files


def statistics(values):
    ordered = sorted(values)
    count = len(ordered)
    if not count:
        raise ValueError('No observations in selected sample')

    def quantile(p):
        position = (count - 1) * p
        low = int(position)
        high = min(low + 1, count - 1)
        return (ordered[low] + (ordered[high] - ordered[low]) * (position - low)) / MIB

    total = sum(ordered)
    return {
        'count': count,
        'sum_bytes': total,
        'mean_bytes': total / count,
        'mean_MiB': total / count / MIB,
        'median_MiB': quantile(0.5),
        'p90_MiB': quantile(0.9),
        'p95_MiB': quantile(0.95),
        'p99_MiB': quantile(0.99),
        'max_MiB': ordered[-1] / MIB,
    }


def resolver_statistics(path):
    samples = {name: array('q') for name in ('all', '3000000_to_6000000_inclusive', 'from_3000000')}
    bounds = {name: [None, None] for name in samples}
    previous = 0
    missing = 0
    trie_sizes = set()
    with path.open(newline='') as stream:
        for row in csv.DictReader(stream):
            block = int(row['blockNum'])
            size = int(row['witnessesSize'])
            if block <= previous or size < 0:
                raise ValueError('Resolver CSV must have increasing block numbers and nonnegative sizes')
            missing += block - previous - 1
            previous = block
            trie_sizes.add(int(row['maxTrieSize']))
            selections = {
                'all': True,
                '3000000_to_6000000_inclusive': 3_000_000 <= block <= 6_000_000,
                'from_3000000': block >= 3_000_000,
            }
            for name, selected in selections.items():
                if selected:
                    samples[name].append(size)
                    if bounds[name][0] is None:
                        bounds[name][0] = block
                    bounds[name][1] = block
    return {
        'samples': {name: {**statistics(values), 'first_last_block': bounds[name]}
                    for name, values in samples.items()},
        'maxTrieSize_values': sorted(trie_sizes),
        'missing_block_numbers_in_full_file_range': missing,
        'missing_block_policy': 'not counted as zero-size observations',
    }


def kv_statistics(files):
    values_by_format = {}
    file_details = {}
    for name in ('opcode', 'kv_compressed', 'kv_uncompressed'):
        values = array('q', [-1]) * (KV_LAST - KV_FIRST + 1)
        rows = outside = duplicates = 0
        with files[name].open(newline='') as stream:
            for row in csv.DictReader(stream):
                rows += 1
                block = int(row['BlockNumber'])
                size = int(row['BlockWitnessSize'])
                if size < 0:
                    raise ValueError(f'{name}: negative witness size')
                if not KV_FIRST <= block <= KV_LAST:
                    outside += 1
                    continue
                previous = values[block - KV_FIRST]
                if previous != -1:
                    duplicates += 1
                    if previous != size:
                        raise ValueError(f'{name}: conflicting witness sizes for block {block}')
                values[block - KV_FIRST] = size
        values_by_format[name] = values
        file_details[name] = {
            'file_rows': rows,
            'rows_outside_requested_range': outside,
            'repeated_block_rows_removed_in_requested_range': duplicates,
        }
    common = array('I', (offset for offset in range(KV_LAST - KV_FIRST + 1)
                        if all(values[offset] != -1 for values in values_by_format.values())))
    if not common:
        raise ValueError('No common blocks in KV comparison')
    results = {name: statistics(array('q', (values[offset] for offset in common)))
               for name, values in values_by_format.items()}
    compressed_mean = results['kv_compressed']['mean_bytes']
    return {
        'requested_inclusive_range': [KV_FIRST, KV_LAST],
        'common_inclusive_range': [common[0] + KV_FIRST, common[-1] + KV_FIRST],
        'common_unique_blocks': len(common),
        'missing_blocks_within_common_bounds': common[-1] - common[0] + 1 - len(common),
        'file_details': file_details,
        'statistics': results,
        'compressed_mean_reduction_vs_opcode_percent':
            100 * (1 - compressed_mean / results['opcode']['mean_bytes']),
        'compressed_mean_reduction_vs_uncompressed_percent':
            100 * (1 - compressed_mean / results['kv_uncompressed']['mean_bytes']),
    }


def main():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('--data-root', type=Path, required=True, help='Directory containing the two repository data directories')
    args = parser.parse_args()
    files = checked_files(args.data_root)
    result = {
        'sources': SOURCES,
        'all_source_hashes_verified': True,
        'bytes_per_MiB': MIB,
        'quantile_method': 'linear interpolation at (count - 1) * percentile',
        'resolver': resolver_statistics(files['resolver']),
        'kv': kv_statistics(files),
    }
    print(json.dumps(result, indent=2))


if __name__ == '__main__':
    main()