"""Reproduce the November 2026 storage experiment. Python 3.12+, FFmpeg on PATH.

Install dependencies from requirements.txt. Run: python benchmark.py --output work
Generated samples are synthetic, not a representative customer corpus.
"""
import argparse
import bz2
import csv
import gzip
import hashlib
import importlib.metadata
import io
import json
import lzma
import math
import platform
import random
import statistics
import struct
import subprocess
import sys
import time
import zipfile
from collections import Counter
from pathlib import Path

CODECS = ['gzip-6', 'gzip-9', 'bzip2-9', 'xz-6', 'xz-9', 'brotli-5',
          'brotli-11', 'zstd-3', 'zstd-19', 'zstd-22', 'lz4-0',
          'blocks-4096', 'blocks-65536', 'blocks-1048576']


def pack_blocks(data, size):
    """Toy indexed container: independent zstd-3 frames or original bytes.

    Existing techniques, no new compression algorithm. Includes all index bytes.
    Format: magic, block size, original size, block count; then <offset, size,
    compressed?> records. Offsets point into the payload after the index.
    """
    import zstandard as zstd
    entries, payload = [], bytearray()
    compressor = zstd.ZstdCompressor(level=3)
    for start in range(0, len(data), size):
        block = data[start:start + size]
        trial = compressor.compress(block)
        compressed = len(trial) < len(block)
        saved = trial if compressed else block
        entries.append(struct.pack('<QIB', len(payload), len(saved), compressed))
        payload.extend(saved)
    return (struct.pack('<4sIQI', b'BC01', size, len(data), len(entries))
            + b''.join(entries) + payload)


def unpack_blocks(blob, only=None):
    import zstandard as zstd
    magic, size, total, count = struct.unpack_from('<4sIQI', blob)
    if magic != b'BC01':
        raise ValueError('Unknown format')
    base = 20 + 13 * count
    output = bytearray()
    for index in (range(count) if only is None else [only]):
        offset, length, compressed = struct.unpack_from('<QIB', blob, 20 + 13 * index)
        block = blob[base + offset:base + offset + length]
        output.extend(zstd.ZstdDecompressor().decompress(block) if compressed else block)
    if only is None and len(output) != total:
        raise ValueError('Wrong output size')
    return bytes(output)


def codec(name):
    import brotli
    import lz4.frame
    import zstandard as zstd
    family, level = name.split('-')
    level = int(level)
    if family == 'gzip':
        return lambda d: gzip.compress(d, compresslevel=level, mtime=0), gzip.decompress
    if family == 'bzip2':
        return lambda d: bz2.compress(d, compresslevel=level), bz2.decompress
    if family == 'xz':
        return lambda d: lzma.compress(d, preset=level), lzma.decompress
    if family == 'brotli':
        return lambda d: brotli.compress(d, quality=level), brotli.decompress
    if family == 'zstd':
        return lambda d: zstd.ZstdCompressor(level=level).compress(d), zstd.ZstdDecompressor().decompress
    if family == 'lz4':
        return lambda d: lz4.frame.compress(d, compression_level=level), lz4.frame.decompress
    return lambda d: pack_blocks(d, level), unpack_blocks


def worker(file, name):
    import psutil
    data = Path(file).read_bytes()
    encode, decode = codec(name)
    process = psutil.Process()
    baseline = process.memory_info().rss
    enc_times, dec_times = [], []
    # One untimed warm-up, then three in-process measurements. File I/O excluded.
    blob = encode(data)
    assert decode(blob) == data
    for _ in range(3):
        start = time.perf_counter_ns()
        blob = encode(data)
        enc_times.append((time.perf_counter_ns() - start) / 1e6)
        start = time.perf_counter_ns()
        restored = decode(blob)
        dec_times.append((time.perf_counter_ns() - start) / 1e6)
        assert hashlib.sha256(restored).digest() == hashlib.sha256(data).digest()
    if name.startswith('blocks-'):
        size = int(name.split('-')[1])
        for i in range(math.ceil(len(data) / size)):
            assert unpack_blocks(blob, i) == data[i * size:(i + 1) * size]
    memory = process.memory_info()
    print(json.dumps(dict(file=Path(file).name, codec=name, original_bytes=len(data),
                          compressed_bytes=len(blob), percent=100 * len(blob) / len(data),
                          encode_ms=statistics.median(enc_times),
                          decode_ms=statistics.median(dec_times),
                          encode_runs_ms=enc_times, decode_runs_ms=dec_times,
                          baseline_rss_bytes=baseline,
                          process_peak_working_set_bytes=getattr(memory, 'peak_wset', None),
                          sha256=hashlib.sha256(data).hexdigest(), verified=True)))


GUIDE = '''A cloud file still needs a home

When a file appears in a folder, that does not tell you where its bytes live.
A local file may occupy the internal disk. A placeholder may contain a name,
a size and a reference to an object stored somewhere else. The application
must retrieve the bytes before it can use them. A fast network can hide some
of this wait, but a disconnected computer cannot fetch a missing block.

Video players can read a small part of a recording and start showing frames.
They need enough container metadata to find the requested section. An editor
that scans an entire recording can cause a very different access pattern.
Scrubbing backwards and forwards also changes which blocks are useful.
Measure cold opens, repeated opens and seeking separately.

Compression replaces predictable data with a shorter description. Repeated
words, recurring field names and similar nearby pixels provide useful clues.
The decoder needs the same dictionary, prediction rules or model weights.
Saving those rules somewhere else does not make their storage cost disappear.
An unfamiliar input may leave little for the model to predict.

Lossless archives restore every original byte. A different image encoding can
restore the same pixels without restoring the source file's metadata or byte
layout. A lossy encoder changes the signal to reduce its size further. These
three guarantees answer different questions. State which one a test checks.

An encrypted object should hide the patterns in the original file. Compress
the plaintext first when the design permits it, then encrypt the result.
Keep the key separate from the stored ciphertext. Recovery procedures matter
because a lost key can make an otherwise intact archive unusable.

A storage quota is a promise about capacity, rather than a measurement of
bytes used today. Compare monthly subscriptions using the same billing term
and currency. Keep the cost of restoring an archive in the calculation.
Ask how long an export takes on the connection that will actually carry it.

For this test, the samples are deliberately small and synthetic. A smooth
gradient favours prediction. Random noise frustrates it. A production archive
contains a mixture whose balance depends on its owner. Test your own mixture
before turning this example into a purchasing or infrastructure decision.
'''


def generate(out):
    from PIL import Image, ImageDraw
    from reportlab.pdfgen import canvas
    samples = out / 'samples'
    samples.mkdir(parents=True, exist_ok=True)
    (samples / 'guide.txt').write_text(GUIDE, encoding='utf-8', newline='\n')
    rng = random.Random(20261103)
    logs = [json.dumps(dict(timestamp=f'2026-11-03T{(i // 3600) % 24:02}:{i // 60 % 60:02}:{i % 60:02}Z',
                            request=f'{rng.getrandbits(64):016x}', route=rng.choice(['/files', '/upload', '/account', '/share']),
                            status=rng.choice([200, 200, 200, 201, 404, 503]),
                            bytes=rng.randrange(200, 2000000), duration_ms=rng.randrange(1, 800)))
            for i in range(6000)]
    (samples / 'events.jsonl').write_text('\n'.join(logs) + '\n', encoding='utf-8', newline='\n')
    w, h = 768, 512
    image = Image.new('RGB', (w, h))
    image.putdata([(x * 255 // w, y * 255 // h, (x + y) * 255 // (w + h))
                   for y in range(h) for x in range(w)])
    draw = ImageDraw.Draw(image)
    draw.rectangle((40, 40, 400, 200), fill='white', outline='black', width=3)
    draw.text((65, 65), 'Synthetic storage sample', fill='black')
    draw.ellipse((480, 260, 700, 480), fill='#eb7525')
    image.save(samples / 'gradient.ppm')
    image.save(samples / 'gradient.png', compress_level=9)
    image.save(samples / 'gradient.jpg', quality=90)
    noise = Image.frombytes('RGB', (512, 512), rng.randbytes(512 * 512 * 3))
    noise.save(samples / 'noise.png', compress_level=9)
    (samples / 'random.bin').write_bytes(rng.randbytes(512 * 1024))
    for compressed in (False, True):
        path = samples / ('report-compressed.pdf' if compressed else 'report-plain.pdf')
        pdf = canvas.Canvas(str(path), pagesize=(595, 842), pageCompression=int(compressed), invariant=1)
        pdf.setTitle('Synthetic cloud storage report')
        for page in range(1, 9):
            pdf.setFont('Helvetica-Bold', 18)
            pdf.drawString(48, 786, f'Storage test report: page {page}')
            pdf.setFont('Helvetica', 10)
            for i, line in enumerate(GUIDE.splitlines()[2:]):
                pdf.drawString(48, 752 - i * 12, line)
            pdf.drawImage(str(samples / 'gradient.jpg'), 48, 90, width=300, height=200)
            pdf.showPage()
        pdf.save()
    command = ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-y', '-f', 'lavfi',
               '-i', 'testsrc2=size=640x360:rate=24:duration=6', '-an',
               '-c:v', 'libx264', '-preset', 'medium', '-crf', '23', '-threads', '1',
               '-pix_fmt', 'yuv420p', '-movflags', '+faststart', str(samples / 'motion.mp4')]
    subprocess.run(command, check=True)
    (out / 'generation.json').write_text(json.dumps(dict(seed=20261103, ffmpeg_command=command,
        ffmpeg_version=subprocess.check_output(['ffmpeg', '-version'], text=True).splitlines()[0]), indent=2))
    return samples


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--output', type=Path, default=Path('work'))
    parser.add_argument('--worker', nargs=2, metavar=('FILE', 'CODEC'))
    args = parser.parse_args()
    if args.worker:
        worker(*args.worker)
        return
    out = args.output.resolve()
    samples = generate(out)
    results = []
    for file in sorted(samples.iterdir()):
        for name in CODECS:
            print(f'{file.name}: {name}', flush=True)
            completed = subprocess.run([sys.executable, __file__, '--worker', str(file), name],
                                       capture_output=True, text=True, check=True, timeout=180)
            results.append(json.loads(completed.stdout))
            (out / 'results.json').write_text(json.dumps(results, indent=2))
    fields = [k for k in results[0] if not k.endswith('_runs_ms')]
    with (out / 'results.csv').open('w', newline='') as stream:
        writer = csv.DictWriter(stream, fieldnames=fields, extrasaction='ignore')
        writer.writeheader()
        writer.writerows(results)
    import psutil
    metadata = dict(python=sys.version, platform=platform.platform(), processor=platform.processor(),
                    cpu_logical=psutil.cpu_count(), ram_bytes=psutil.virtual_memory().total,
                    packages={p: importlib.metadata.version(p) for p in
                              ['pillow', 'reportlab', 'zstandard', 'brotli', 'lz4', 'psutil']},
                    zlib=__import__('zlib').ZLIB_RUNTIME_VERSION,
                    repeats=3, warmups=1, execution='Sequential, one fresh process per file/codec',
                    memory='OS process peak working set across warm-up, encode, decode and verification; includes interpreter and input, not codec-only allocation.',
                    timing='Codec calls only; excludes disk I/O, process startup and hash verification.')
    (out / 'environment.json').write_text(json.dumps(metadata, indent=2))
    manifest = [{'file': p.name, 'bytes': p.stat().st_size, 'sha256': hashlib.sha256(p.read_bytes()).hexdigest()}
                for p in sorted(samples.iterdir())]
    (out / 'manifest.json').write_text(json.dumps(manifest, indent=2))
    with zipfile.ZipFile(out / 'samples.zip', 'w', compression=zipfile.ZIP_DEFLATED) as archive:
        for p in sorted(samples.iterdir()):
            archive.write(p, p.name)
        archive.write(out / 'manifest.json', 'manifest.json')
    print(f'Verified {len(results)} file/codec pairs. Results: {out}', flush=True)


if __name__ == '__main__':
    main()