""" Run this on your own machine before you submit. It runs the SAME checks the server runs, so if it passes here it will not be rejected there. .venv/bin/python competition/selftest.py --dir path/to/your/submission Your submission directory holds either model.py + weights.safetensors (track B) or drawer.onnx (track A) and this figures out which. A preview PNG is written next to it so you can look at what your model actually draws -- worth doing, because every check here can pass on a picture no human would mistake for a doodle. Exit code 0 means accepted. Anything else prints why, in the same words the server would use. """ from __future__ import annotations import argparse import ast import sys import time from pathlib import Path import numpy as np sys.path.insert(0, str(Path(__file__).resolve().parent)) from spec import (DETERMINISM_TOL, EMBED_DIM, IMAGE_SIZE, IN_EMBED, IN_NOISE, MAX_SECONDS_PER_IMAGE, NOISE_SHAPE, OUT_IMAGE, Problem, check_output, validate_graph, validate_signature) # Imports that will not work in the sandbox. Flagged as a courtesy so you find # out here rather than from a timeout on the server. This is a LINT, not a # security control -- the container is what actually contains anything, and # anyone determined can evade an AST scan trivially. UNAVAILABLE = {"requests", "urllib", "urllib3", "httpx", "aiohttp", "socket", "http", "ftplib", "telnetlib", "smtplib", "huggingface_hub"} def lint_source(path: Path) -> list[Problem]: """Parse model.py and warn about things the sandbox will refuse.""" problems: list[Problem] = [] try: tree = ast.parse(path.read_text()) except SyntaxError as e: return [Problem("model.py", f"syntax error on line {e.lineno}: {e.msg}")] for node in ast.walk(tree): mods = [] if isinstance(node, ast.Import): mods = [a.name for a in node.names] elif isinstance(node, ast.ImportFrom) and node.module: mods = [node.module] for m in mods: if m.split(".")[0] in UNAVAILABLE: problems.append(Problem( "imports", f"{m!r} needs network access, which the sandbox " "disables. Put everything in your weights file")) names = {n.name for n in ast.walk(tree) if isinstance(n, ast.ClassDef)} if "Drawer" not in names: problems.append(Problem( "model.py", f"must define `class Drawer(RoguesDrawer)`; found {sorted(names) or 'no classes'}")) return problems def fixed_inputs(seed: int = 1234, tokens: int = 77): """The same inputs the server uses, so results here match results there.""" rng = np.random.default_rng(seed) return (rng.standard_normal((1, tokens, EMBED_DIM), dtype=np.float32), rng.standard_normal(NOISE_SHAPE).astype(np.float32)) def run_python(d: Path): problems = lint_source(d / "model.py") if problems: return None, problems sys.path.insert(0, str(d)) sys.path.insert(0, str(Path(__file__).resolve().parent)) import importlib mod = importlib.import_module("model") drawer = getattr(mod, "Drawer")() weights = {} wpath = d / "weights.safetensors" if wpath.exists(): from safetensors.numpy import load_file weights = load_file(str(wpath)) drawer.load(weights) def call(): e, n = fixed_inputs() return np.asarray(drawer.draw(e, n), dtype=np.float32) return call, [] def run_onnx(d: Path): import onnx import onnxruntime as ort path = d / "drawer.onnx" problems = validate_graph(onnx.load(str(path))) if problems: return None, problems so = ort.SessionOptions() so.log_severity_level = 3 sess = ort.InferenceSession(str(path), so, providers=["CPUExecutionProvider"]) problems = validate_signature(sess) if problems: return None, problems def call(): e, n = fixed_inputs() return np.asarray(sess.run([OUT_IMAGE], {IN_EMBED: e, IN_NOISE: n})[0], dtype=np.float32) return call, [] def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--dir", required=True, help="your submission directory") ap.add_argument("--preview", default="preview.png") args = ap.parse_args() d = Path(args.dir) if (d / "model.py").exists(): track, build = "B (model.py + safetensors)", run_python elif (d / "drawer.onnx").exists(): track, build = "A (onnx)", run_onnx else: print(f" no submission found in {d}\n" f" expected model.py + weights.safetensors, or drawer.onnx") return 2 print(f" track {track}") try: call, problems = build(d) except Exception as e: # a student's own import error print(f" FAILED to load: {type(e).__name__}: {e}") return 1 if problems: print(" REJECTED") for p in problems: print(f" - {p}") return 1 t0 = time.time() try: img = call() except Exception as e: print(f" FAILED during draw: {type(e).__name__}: {e}") return 1 elapsed = time.time() - t0 if img.ndim == 3 and img.shape[0] == 1: img = img[0] problems = list(check_output(img)) if elapsed > MAX_SECONDS_PER_IMAGE: problems.append(Problem( "speed", f"{elapsed:.0f}s per image exceeds the {MAX_SECONDS_PER_IMAGE:.0f}s limit")) again = call() if again.ndim == 3 and again.shape[0] == 1: again = again[0] drift = float(np.abs(again - img).max()) if drift > DETERMINISM_TOL: problems.append(Problem( "determinism", f"identical inputs gave different images (drift {drift:.2e}). " "All randomness must come from `noise`")) from PIL import Image Image.fromarray((np.clip(img, 0, 1) * 255).astype(np.uint8)).save(d / args.preview) if problems: print(" REJECTED") for p in problems: print(f" - {p}") return 1 print(f" ACCEPTED") print(f" {elapsed:.2f}s per image ink fraction {float((img < 0.5).mean()):.4f}") print(f" determinism drift {drift:.1e}") print(f" preview written to {d / args.preview} — look at it before submitting") return 0 if __name__ == "__main__": raise SystemExit(main())