Starting Model Test on Art Theory Category
Starting Model Test on Art Theory Category
Agent
Pi
Model
GPT-5.3 Codex
StartedApr 11, 4:47 AM
Shared
Apr 11, 4:49 AM
Messages60
Downloads18
@setup.txt
we're gonna start with the art_theory category of MMMU. Lets start by running the model against the dev set
>ls -la>find .. -maxdepth 4 -iname '*mmmu*' -print>curl -s http://127.0.0.1:8080/health || true>python - <<'PY'
from datasets import load_dataset
# try loading specific config
try:
ds=load_dataset('MMMU/MMMU','Art_Theory',split='dev')
except Exception as e:
print('err1',e)
ds=load_dataset('MMMU/MMMU',split='dev')
print(ds)
raise
print(ds)
print(ds.features)
print('len',len(ds))
print(ds[0].keys())
for k,v in ds[0].items():
if k.startswith('image'):
print(k, type(v), getattr(v,'size',None))
else:
print(k,repr(v)[:200])
PY>python3 - <<'PY'
from datasets import load_dataset
try:
ds=load_dataset('MMMU/MMMU','Art_Theory',split='dev')
except Exception as e:
print('err1',e)
ds=load_dataset('MMMU/MMMU',split='dev')
print(ds)
raise
print(ds)
print(ds.features)
print('len',len(ds))
print(ds[0].keys())
for k,v in ds[0].items():
if k.startswith('image'):
print(k, type(v), getattr(v,'size',None))
else:
print(k,repr(v)[:200])
PY>hf --help | head>hf repo-files MMMU/MMMU --repo-type dataset | head -n 80>hf repo-files -h>hf download MMMU/MMMU --repo-type dataset --dry-run | head -n 120>hf download MMMU/MMMU --repo-type dataset --include 'Art_Theory/dev-00000-of-00001.parquet'>python3 - <<'PY'
import pyarrow.parquet as pq
path='/Users/mukund/.cache/huggingface/hub/datasets--MMMU--MMMU/snapshots/21d1d90a93c7450d30bddb579d7b510c00b8a9ab/Art_Theory/dev-00000-of-00001.parquet'
t=pq.read_table(path)
print(t.schema)
print('rows',t.num_rows)
print('cols',t.num_columns)
row=t.slice(0,1).to_pylist()[0]
print(row.keys())
for k,v in row.items():
if isinstance(v,(bytes,bytearray)):
print(k,'bytes',len(v))
elif isinstance(v,dict):
print(k,'dict',v.keys())
elif isinstance(v,list):
print(k,'list',len(v), type(v[0]).__name__ if v else None)
else:
print(k,type(v).__name__,str(v)[:140])
PY>which duckdb || which parquet-tools || which sqlite3>python3 -m pip install --user datasets pyarrow pillow requests -q>python3 -m venv .venv && . .venv/bin/activate && pip install -q datasets pyarrow pillow requests>. .venv/bin/activate && python - <<'PY'
from datasets import load_dataset
ds=load_dataset('MMMU/MMMU','Art_Theory',split='dev')
print(ds)
print('len',len(ds))
print(ds.features)
r=ds[0]
print('keys',r.keys())
for k,v in r.items():
if k.startswith('image'):
print(k, type(v), getattr(v,'size',None))
else:
s=str(v)
print(k, s[:200].replace('\n',' '))
PY>. .venv/bin/activate && python - <<'PY'
import base64, io, json, requests
from datasets import load_dataset
def pil_to_data_url(img):
buf=io.BytesIO(); img.save(buf, format='PNG'); b=base64.b64encode(buf.getvalue()).decode('utf-8')
return f'data:image/png;base64,{b}'
ds=load_dataset('MMMU/MMMU','Art_Theory',split='dev')
r=ds[0]
question=r['question']
opts=r['options']
text=question+'\n'+'\n'.join([f"{chr(65+i)}. {o}" for i,o in enumerate(opts)])+'\nAnswer with only the letter (A, B, C, or D).'
content=[{"type":"text","text":text}]
for i in range(1,8):
img=r[f'image_{i}']
if img is not None:
content.append({"type":"image_url","image_url":{"url":pil_to_data_url(img)}})
payload={"model":"local","messages":[{"role":"user","content":content}],"temperature":0}
resp=requests.post('http://127.0.0.1:8080/v1/chat/completions',json=payload,timeout=120)
print(resp.status_code)
print(resp.text[:1000])
PY>. .venv/bin/activate && python - <<'PY'
import base64, io, re, requests
from datasets import load_dataset
SERVER='http://127.0.0.1:8080/v1/chat/completions'
# verify split sizes
for split in ['dev','validation','test']:
ds_split=load_dataset('MMMU/MMMU','Art_Theory',split=split)
print(f'Art_Theory {split}: {len(ds_split)}')
ds=load_dataset('MMMU/MMMU','Art_Theory',split='dev')
pat=re.compile(r'\b([A-Z])\b')
letters='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
def pil_to_data_url(img):
buf=io.BytesIO(); img.save(buf, format='PNG')
return 'data:image/png;base64,'+base64.b64encode(buf.getvalue()).decode('utf-8')
correct=0
for idx,row in enumerate(ds):
options=row['options']
opt_lines=[f"{letters[i]}. {opt}" for i,opt in enumerate(options)]
allowed=', '.join(letters[i] for i in range(len(options)))
text=(
"Answer the multiple-choice question based on the provided image(s).\n"
"Return ONLY the single option letter and nothing else.\n\n"
f"Question: {row['question']}\n"
"Options:\n" + "\n".join(opt_lines) + "\n\n"
f"Valid letters: {allowed}."
)
content=[{"type":"text","text":text}]
for i in range(1,8):
img=row[f'image_{i}']
if img is not None:
content.append({"type":"image_url","image_url":{"url":pil_to_data_url(img)}})
payload={"model":"local","messages":[{"role":"user","content":content}],"temperature":0,"max_tokens":8}
r=requests.post(SERVER,json=payload,timeout=240)
r.raise_for_status()
out=r.json()['choices'][0]['message']['content'].strip()
m=pat.search(out)
pred=m.group(1) if m else out[:1].upper()
gold=row['answer'].strip().upper()
ok=(pred==gold)
correct+=ok
print(f"{idx+1}/{len(ds)} id={row['id']} pred={pred!r} raw={out!r} gold={gold!r} {'OK' if ok else 'WRONG'}")
print(f"Accuracy: {correct}/{len(ds)} = {correct/len(ds):.1%}")
PY>. .venv/bin/activate && python - <<'PY'
from datasets import load_dataset
letters='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
ds=load_dataset('MMMU/MMMU','Art_Theory',split='dev')
for row in ds:
opts=row['options']
print(row['id'],type(opts),opts, 'len',len(opts))
if len(opts)>26:
print('too many')
for i,opt in enumerate(opts):
if i>=len(letters):
print('i too high',i)
PY>. .venv/bin/activate && python - <<'PY'
import base64, io, re, ast, requests
from datasets import load_dataset
SERVER='http://127.0.0.1:8080/v1/chat/completions'
letters='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
pat=re.compile(r'\b([A-Z])\b')
def parse_options(o):
if isinstance(o,list):
return o
if isinstance(o,str):
try:
v=ast.literal_eval(o)
if isinstance(v,list):
return [str(x) for x in v]
except Exception:
pass
raise ValueError(f'Unexpected options format: {type(o)} {o!r}')
def pil_to_data_url(img):
buf=io.BytesIO(); img.save(buf, format='PNG')
return 'data:image/png;base64,'+base64.b64encode(buf.getvalue()).decode('utf-8')
for split in ['dev','validation','test']:
print(f'Art_Theory {split}:', len(load_dataset('MMMU/MMMU','Art_Theory',split=split)))
ds=load_dataset('MMMU/MMMU','Art_Theory',split='dev')
correct=0
for idx,row in enumerate(ds):
options=parse_options(row['options'])
opt_lines=[f"{letters[i]}. {opt}" for i,opt in enumerate(options)]
allowed=', '.join(letters[i] for i in range(len(options)))
text=(
"Answer the multiple-choice question based on the provided image(s).\n"
"Return ONLY the single option letter and nothing else.\n\n"
f"Question: {row['question']}\n"
"Options:\n" + "\n".join(opt_lines) + "\n\n"
f"Valid letters: {allowed}."
)
content=[{"type":"text","text":text}]
for i in range(1,8):
img=row[f'image_{i}']
if img is not None:
content.append({"type":"image_url","image_url":{"url":pil_to_data_url(img)}})
payload={"model":"local","messages":[{"role":"user","content":content}],"temperature":0,"max_tokens":8}
resp=requests.post(SERVER,json=payload,timeout=240)
resp.raise_for_status()
out=resp.json()['choices'][0]['message']['content'].strip()
m=pat.search(out)
pred=(m.group(1) if m else out[:1].upper())
gold=row['answer'].strip().upper()
ok=(pred==gold)
correct+=ok
print(f"{idx+1}/{len(ds)} {row['id']} pred={pred} gold={gold} raw={out!r} {'OK' if ok else 'WRONG'}")
print(f"Accuracy: {correct}/{len(ds)} = {correct/len(ds):.1%}")
PY>. .venv/bin/activate && python run_mmmu_eval.py --subject Art_Theory --split devDone — I ran LFM2.5-VL-1.6B on MMMU / Art_Theory / dev against your local llama-server.
Art_Theory split sizes:
dev: 5validation: 30test: 429dev_Art_Theory_1 → pred A, gold D ❌dev_Art_Theory_2 → pred A, gold A ✅dev_Art_Theory_3 → pred B, gold A ❌dev_Art_Theory_4 → pred A, gold B ❌dev_Art_Theory_5 → pred C, gold C ✅Accuracy: 2/5 = 40.0%
I also saved a reusable script at:
run_mmmu_eval.pyRun it with:
. .venv/bin/activatepython run_mmmu_eval.py --subject Art_Theory --split devIf you want, I can next run the same on validation for this category and give a fuller breakdown.
can you write a requirements file
Added requirements.txt at:
requirements.txtContents:
datasetspyarrowpillowrequestsIf you want, I can also add a short README snippet with venv setup/install/run commands.
git commit and continue
>git status --short>git add requirements.txt run_mmmu_eval.py && git commit -m "Add MMMU Art_Theory eval script and Python requirements">uv run python run_mmmu_eval.py --subject Art_Theory --split devImplemented and committed ✅
43cdacbLog MMMU eval runs, subject results, and predictions to SQLiterun_mmmu_eval.py--db-path (default: mmmu_runs.db)runs (one row per script invocation)subject_results (per-subject aggregate)predictions (per-question detail)running → completed or failedAt the end of each run it prints:
runs, subject_results, predictions)So you get immediate feedback on how much data was written.
add better error handling if the backend server is not reachable. keep it concise tho
don't bloat my code dawg
. .venv/bin/activatepython run_mmmu_eval.py --subject Art_Theory --split dev