diff --git a/.gitattributes b/.gitattributes index 7cfdd5b..04aa34c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,6 @@ results/** -text results/**/*.csv -whitespace results/**/*.svg -whitespace results/**/*.log -whitespace + +# Preserve captured scientific stdout byte-for-byte, including blank final lines. +results/head-dominance/*-output.txt -whitespace diff --git a/README.md b/README.md index 00d8049..346748b 100644 --- a/README.md +++ b/README.md @@ -42,3 +42,12 @@ Full resolved versions and run provenance accompany each result. See the SOP for ## CPU block activation measurements See [measurement definitions](activation/README.md) and [observed activation report](results/block-activation/REPORT.md) for all 24 layers, contiguous output-channel blocks, exact/near-zero rates and output energy concentration. This is an observational SiLU-gated study, not a pruning or reuse quality claim. + +## Adjacent-layer head influence concentration + +The CPU study measures all 23 adjacent transitions, observation-wise effective +head counts and Q/K/V influence shares, with equal-size weak/strong/random joint +removal controls. See [definitions and reproduction](dominance/README.md), +[fixed protocol](dominance/PROTOCOL.md), and [measured report](results/head-dominance/REPORT.md). +These are descriptive intervention effects, not additive contribution probabilities +or validated final-LM pruning gains. diff --git a/dominance/PROTOCOL.md b/dominance/PROTOCOL.md new file mode 100644 index 0000000..76c1cdd --- /dev/null +++ b/dominance/PROTOCOL.md @@ -0,0 +1,76 @@ +# Head influence concentration: measurement protocol + +Target: quantify the statistical distribution of large and small upstream head +influences on the next layer's Q/K/V; identity of dominant heads may vary at each +sample, query token and recipient head. No training, GPU, deletion or model edits. + +Scope: all 23 adjacent layer transitions of the retained Qwen3.5-2B text model, +including 16-head linear attention and 8-head full attention. Measure effective +Q/K/V, and retrieval probability changes where the recipient uses full attention. +The intervening source-layer residual/MLP and recipient input normalization must +be recomputed after interventions. Residual stream is preserved: removing an +attention head is not removing all information reaching the recipient. + +For each observation, remove one upstream attention head's projected contribution +at a time. Record nonnegative influence magnitude d_i and its absolute scale. +When sum(d_i)>0, p_i=d_i/sum(d_i) is normalized intervention influence, NOT an +additive causal attribution or a native attention probability over upstream heads. +All-zero influence has undefined concentration and must be flagged, not assigned +uniform or concentrated values. Record small total influences explicitly. + +Primary continuous measures: +- Effective head count: N_eff=1/sum(p_i^2), range [1,H]. +- Effective head fraction: N_eff/H, supports different H. +- Normalized concentration: C=(sum(p_i^2)-1/H)/(1-1/H), range [0,1]. +These are related summaries, not independent evidence. Report distributions over +texts/tokens/recipient heads/layers, not only concentration of globally averaged +influences. Primary query positions 7 onward avoid trivial short-prefix attention. + +Secondary intuitive measures: top1/top25%/top50% influence share, bottom25% +share, and number of heads reaching 90% influence. Preserve raw influence tables. +No RMS activity threshold is used to define influence concentration. + +Joint interventions are required: per observation's weak-head set and strong-head +set must be removed together and measured, because individual ablations are not +additive and may conceal cancellation or synergy. Rankings may be input-specific; +these experiments cannot imply a universally removable fixed head set. Distinguish +local next-layer QKV/probability effects from unmeasured final LM quality. + +CPU measurements, scientific plots, tests and audits execute on the previously +authorized server 8.211.154.134. All weights and existing caches remain retained. + +## Fixed execution details (before real-model measurements) + +Four texts are reused from crossqkv/fresh_samples.json (two Chinese/two English), +32 tokens each; no independent generalization claim. All tokens are computed, +queries7..31 are primary statistical observations. The observer caches source +residual inputs and exact per-head projected output components; their sum is +checked against the native branch output. First text's full forward is checked +against the canonical streaming forward, and every transition has a null-mask +control. No cached tensor is deleted. + +For each query/recipient query-head, individual influence is the L2 change divided +by that same baseline vector's L2 (floor1e-12). Q,K,V are analyzed separately. +A declared balanced score sqrt((relative_Q^2+relative_K^2+relative_V^2)/3) is also +analyzed; it does not treat their different physical scales as directly additive. +Full attention: Q/K are measured after learned norm/partial RoPE, V before output +mixing, and actual softmax retrieval JS is measured. Linear attention: Q/K/V are +after causal convolution+SiLU; Q/K include the delta-rule L2 normalization and Q +scale. It has no ordinary softmax retrieval distribution. Full-attention GQA K/V +are repeated to query-head slots for aligned joint tests; these slots are not +independent KV heads, and statistical observations are not independent samples. + +Joint weak-half/strong-half/random-half/all-head removal uses a separate coherent +sequence for each recipient query-head. At each source token, choose the donor +half by that recipient query-head's QKV influence ranking at the same token. +Recompute full recipient sequence; convolution and QK retrieval can therefore +propagate changes between positions. Only the intended recipient head is scored +for each scenario. This is an input-adaptive diagnostic, not a computationally +free head-selection policy. Random uses fixed seed31000+sample_index*100+layer. +Compare joint effects to sums of corresponding single effects but do not assume +additivity. For JS use the same joint sets (selected by QKV, not by JS). + +No success criterion assumes that dominance exists. Report effective-head and +absolute-effect distributions even if contributions are diffuse. No final-LM +NLL or deployable pruning quality claim. Scientific output includes all raw +single/joint effects and deletion masks; large baseline caches remain external. diff --git a/dominance/README.md b/dominance/README.md new file mode 100644 index 0000000..0feda3f --- /dev/null +++ b/dominance/README.md @@ -0,0 +1,75 @@ +# Statistical head influence dominance (CPU) + +This study answers how concentrated the **distribution of measured upstream-head +influences** is, without assuming that dominant head identities are fixed across +texts, positions or recipient heads. Read [registered protocol](PROTOCOL.md). + +- `core.py`: normalized effective Q/K/V extraction for both attention types, + source projected head decomposition, exact nonlinear downstream recomputation, + coherent masks, continuous concentration and JS. +- `run.py`: four fixed prior texts, all23 adjacent connections, single-head and + weak/strong/random-half/all-head interventions, per-transition checkpointing, + canonical-forward/null controls, preserved baseline caches. +- `report.py`: full saved-data distributions, per-transition CSV, continuous + effective head plots, influence rank curves, joint tests and Chinese report. +- `audit.py`: independently recomputes concentration/masks from raw effects, + validates scope/null/head sums, and hashes retained caches, source and results. +- `tests/test_dominance.py`: continuous limiting cases, undefined zero-total, + token-dependent masks, JS identity, both canonical mixed-layer paths, and joint + interventions on a small checkpoint with known arrays. + +Influence for Q/K/V is relative L2 change after deleting one whole donor attention +head (at all sequence positions), preserving the prior residual and recomputing +source MLP + recipient normalization. Source **output/value heads** are the units: +16 for linear attention,8 for full attention. These are not arbitrary individual +Q/K/V weight blocks. For a recipient query-head at one token, normalize donor +influences to p. Primary continuous summary is N_eff=1/sum(p²), and N_eff/H. +C=(sum(p²)-1/H)/(1-1/H) is a related0..1 concentration score. These are not +independent confirmations. No RMS near-zero threshold or R fitting is used. + +Q/K/V are separate primary scales; balanced QKV score is RMS of their three +relative changes. It is a declared summary, not a norm in a single physical space. +Zero total implies undefined concentration, not a uniform distribution. Single-head relative effects before head-share normalization and true joint +effects are retained as well as shares. The saved metric named max_absolute is +the largest unsigned relative-L2 effect, not an unnormalized hidden-space norm. +The total field sums individual relative effects; it is not the measured joint +effect of removing the whole attention branch. + +Full-attention recipients additionally have real token-key probability JS changes; +linear attention does not have an ordinary softmax attention probability matrix. +Full-attention K/V repeat their GQA groups to query-head slots for aligned joint +experiments, not as extra independent KV observations. Every headline statistic +is descriptive: query/token/head observations are dependent, and four short prior +texts do not establish universal dominance or independent generalization. + +Joint masks vary per source token and recipient head. Recompute a coherent full +sequence for each recipient head, scoring that intended head afterward. Rank by +single QKV influence, remove weakest half/strongest half/a fixed-seed random half, +or all attention heads. This investigates interactions and cross-token effects; +it is not a validated free selector, fixed removable head subset or LM pruning. + +Only run model work and scientific plotting on the authorized CPU server: +``` +OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 venv/bin/python -B -m unittest tests.test_dominance -v +OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 venv/bin/python -B -m dominance.run --source runs/cpu-qwen35-2b --out runs/NEW_DOMINANCE_DIR --seconds 7200 +venv/bin/python -B -m dominance.report runs/NEW_DOMINANCE_DIR +venv/bin/python -B -m dominance.audit runs/NEW_DOMINANCE_DIR runs/cpu-qwen35-2b +``` + +No training, GPU, new model download or automatic deletion. Raw intervention +arrays and masks are delivered to Git; large baseline caches stay on the server +with path/size/SHA metadata. Existing checkpoints/caches are never removed. + +The saved sum of selected individual effects is only a query-wise mask-weighted +reference: individual ablations remove a head at all positions, whereas joint +masks can vary across positions. Their difference cannot isolate pure nonlinear +interaction, since temporal intervention supports also differ. Use the measured +weak/strong/random joint effects directly; do not call this reference an additive +causal decomposition or a Shapley attribution. + +Scale sensitivity: after reviewing the first complete summaries, add sqrt(JS) +(Jensen–Shannon distance) alongside primary JS divergence. Divergence is roughly +quadratic for small changes, so shares and effective head counts depend on this +scale. All original primary statistics and raw interventions remain unchanged; +primary-analysis-summary.json preserves the pre-sensitivity aggregate for exact +audit comparison. Retrieval-scale plots and source tables show both definitions. diff --git a/dominance/__init__.py b/dominance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dominance/audit.py b/dominance/audit.py new file mode 100644 index 0000000..269a19a --- /dev/null +++ b/dominance/audit.py @@ -0,0 +1,90 @@ +"""Recompute saved concentration, masks, scale and scope checks; preserve caches.""" +import json,hashlib,csv,sys +from pathlib import Path +import numpy as np +from .core import concentration + + +def main(root,source): + root=Path(root);source=Path(source);status=json.loads((root/'status.json').read_text());assert status['state']=='completed' + samples=json.loads((root/'samples.json').read_text());assert len(samples)==4 and all(len(s['token_ids'])==32 for s in samples) + expected={(s['id'],i) for s in samples for i in range(23)};actual={(x['sample'],x['source']) for x in status['completed_links']};assert actual==expected and len(status['completed_links'])==92 + cfg=json.loads((source/'model_config.json').read_text())['text_config'];obs=dict(np.load(root/'observations.npz'));lookup=sorted(s['id'] for s in samples);counts={};undefined={};full_js_links=0;joint_values={} + for sample,layer in sorted(actual): + f=root/'raw'/f'{sample}-link-{layer:02d}.npz';d=dict(np.load(f));hs=8 if cfg['layer_types'][layer]=='full_attention' else 16;hr=8 if cfg['layer_types'][layer+1]=='full_attention' else 16 + if hr==8: + full_js_links+=1;d['single_sqrtJS']=np.sqrt(d['single_JS']) + for label in ('weak_half','strong_half','random_half','all_heads'): + d['joint_'+label+'_sqrtJS']=np.sqrt(d['joint_'+label+'_JS']) + mask=d.get('mask_'+label,np.ones((hr,32,hs),dtype=np.float32)) + d['individual_sum_'+label+'_sqrtJS']=np.sum(d['single_sqrtJS'].transpose(2,1,0)*mask,-1).T + d['null_sqrtJS']=np.sqrt(d['null_JS']) + masks={label:d['mask_'+label] for label in ('weak_half','strong_half','random_half')} + for label,m in masks.items(): + assert m.shape==(hr,32,hs);assert np.isin(m,[0,1]).all();np.testing.assert_array_equal(m.sum(-1),hs//2) + np.testing.assert_array_equal(masks['weak_half']+masks['strong_half'],1) + ranked=np.argsort(d['single_QKV'].transpose(2,1,0),axis=-1,kind='stable') + np.testing.assert_array_equal(np.take_along_axis(masks['weak_half'],ranked[...,:hs//2],-1),1) + for key in ('Q','K','V','QKV')+(('JS','sqrtJS') if hr==8 else ()): + single=d['single_'+key];assert single.shape==(hs,32,hr) and np.isfinite(single).all() and np.min(single)>=0 + effect=single[:,7:,:].transpose(1,2,0);stats=concentration(effect);counts[key]=counts.get(key,0)+effect.shape[0]*hr;undefined[key]=undefined.get(key,0)+int((~stats['valid']).sum()) + choose=(obs[key+'_source']==layer)&(obs[key+'_sample_index']==lookup.index(sample)) + assert int(choose.sum())==25*hr + for metric,values in stats.items(): + if metric=='valid':continue + np.testing.assert_allclose(obs[key+'_'+metric][choose],values.ravel(),rtol=1e-6,atol=1e-7,equal_nan=True) + for label in ('weak_half','strong_half','random_half','all_heads'): + mask=masks.get(label,np.ones((hr,32,hs),dtype=np.float32));expected_sum=np.sum(single.transpose(2,1,0)*mask,-1).T + np.testing.assert_allclose(d['individual_sum_'+label+'_'+key],expected_sum,rtol=1e-6,atol=1e-8) + assert d['joint_'+label+'_'+key].shape==(32,hr) and np.isfinite(d['joint_'+label+'_'+key]).all() + joint_values.setdefault((key,label),[]).extend(d['joint_'+label+'_'+key][7:].ravel()) + assert float(np.max(d['null_'+key]))<=2e-5 + assert full_js_links==24 + summary=json.loads((root/'analysis-summary.json').read_text()) + def check_stats(values,record): + v=np.asarray(values,dtype=np.float64);v=v[np.isfinite(v)];assert len(v)==record['count'] + if len(v): + for name,x in zip(['p10','median','p90','mean'],[*np.quantile(v,[.1,.5,.9]),v.mean()]):np.testing.assert_allclose(record[name],x,rtol=2e-6,atol=1e-8) + for key in counts: + for metric,record in summary['concentration'][key].items():check_stats(obs[key+'_'+metric],record) + primary=json.loads((root/'primary-analysis-summary.json').read_text()) + for section in ('concentration','joint','paired_joint','by_recipient_type'): + for key,value in primary[section].items():assert summary[section][key]==value,(section,key) + full_sources=sorted({x['source'] for x in status['completed_links'] if x['full_attention_recipient']}) + assert summary['full_attention_source_indices']==full_sources + for key,groups in summary['by_recipient_type'].items(): + src=obs[key+'_source'] + for label,metrics in groups.items(): + choose=np.isin(src,full_sources) if label=='full_attention' else ~np.isin(src,full_sources) + for metric,record in metrics.items():check_stats(obs[key+'_'+metric][choose],record) + for (key,label),values in joint_values.items():check_stats(values,summary['joint'][key][label]) + for key in counts: + weak=np.asarray(joint_values[(key,'weak_half')]);strong=np.asarray(joint_values[(key,'strong_half')]);random=np.asarray(joint_values[(key,'random_half')]);valid=weak>0;record=summary['paired_joint'][key] + assert record['zero_weak_effect_count']==int((~valid).sum()) + np.testing.assert_allclose(record['weak_less_than_strong_fraction'],np.mean(weak0 + p=np.divide(d,total[...,None],out=np.zeros_like(d),where=valid[...,None]);square=np.sum(p*p,-1) + neff=np.divide(1,square,out=np.full_like(total,np.nan),where=valid) + ordered=np.sort(p,axis=-1)[...,::-1] + return {'total':total,'max_absolute':d.max(-1),'valid':valid,'effective_heads':neff,'effective_fraction':neff/h, + 'concentration':np.where(valid,(square-1/h)/(1-1/h),np.nan), + 'top1':np.where(valid,ordered[...,0],np.nan),'top25':np.where(valid,ordered[...,:max(1,h//4)].sum(-1),np.nan), + 'top50':np.where(valid,ordered[...,:max(1,h//2)].sum(-1),np.nan), + 'bottom25':np.where(valid,ordered[...,-max(1,h//4):].sum(-1),np.nan), + 'heads90':np.where(valid,1+np.sum(np.cumsum(ordered,axis=-1)<.9,axis=-1),np.nan)} + + +def masks_from_influences(d,weak=True): + """d [donor,T,recipient]; output masks [recipient,T,donor].""" + ranked=np.argsort(d.transpose(2,1,0),axis=-1,kind='stable');n=d.shape[0]//2 + selected=ranked[...,:n] if weak else ranked[...,-n:] + mask=np.zeros_like(ranked,dtype=np.float32);np.put_along_axis(mask,selected,1,axis=-1) + return mask + + +def js_rows(a,b): + a=np.asarray(a,dtype=np.float64);b=np.asarray(b,dtype=np.float64);m=(a+b)/2 + def term(x):return np.where(x>0,x*np.log(np.maximum(x,1e-300)/np.maximum(m,1e-300)),0) + return np.maximum(0,.5*np.sum(term(a)+term(b),axis=-1)) + + +class InfluenceLM(StreamingLM): + capturing=False + def linear(self,x,name,rows=256): + y=super().linear(x,name,rows) + if self.capturing and (name.endswith('self_attn.o_proj') or name.endswith('linear_attn.out_proj')): + layer=int(name.split('.')[1]);nh=self.config['num_attention_heads'] if 'self_attn' in name else self.config['linear_num_value_heads'] + xx=x.reshape(*x.shape[:2],nh,-1);dim=xx.shape[-1];t=self.tensors[PREFIX+name+'.weight'];od,idim=t.shape + out=np.empty((*x.shape[:2],nh,od),dtype=np.float32) + for start in range(0,od,256): + self.guard();stop=min(od,start+256);w=t.read(slice(start*idim,stop*idim)).reshape(stop-start,idim) + for head in range(nh):out[:,:,head,start:stop]=xx[:,:,head]@w[:,head*dim:(head+1)*dim].T + del w;t.evict() + err=float(np.linalg.norm(out.sum(2).astype(np.float64)-y)/max(np.linalg.norm(y.astype(np.float64)),1e-30)) + if err>2e-5:raise AssertionError(('head sum mismatch',layer,err)) + self.head_outputs=out;self.head_sum_errors.append({'layer':layer,'relative_l2':err}) + return y + + def baseline(self,ids,root): + h=self.embedding(ids);self.head_sum_errors=[] + for layer,kind in enumerate(self.config['layer_types']): + raw=h.copy();n=self.normalization(h,f'layers.{layer}.input_layernorm');self.capturing=True + out,_=self.full_attention(n,layer) if kind=='full_attention' else (self.linear_attention(n,layer),None) + self.capturing=False;heads=self.head_outputs;self.head_outputs=None + mid=raw+out;h=mid+self.mlp(self.normalization(mid,f'layers.{layer}.post_attention_layernorm'),layer) + np.savez(root/f'layer-{layer}.npz',raw=raw,out=out,heads=heads,next_raw=h) + print('baseline layer',layer,flush=True) + return h + + def recipient(self,raw,layer): + h=self.normalization(raw,f'layers.{layer}.input_layernorm');c=self.config;b,t=h.shape[:2] + if c['layer_types'][layer]=='full_attention': + p=f'layers.{layer}.self_attn.';nh=c['num_attention_heads'];nk=c['num_key_value_heads'];d=c['head_dim'] + q=self.linear(h,p+'q_proj').reshape(b,t,nh,2*d)[...,:d];k=self.linear(h,p+'k_proj').reshape(b,t,nk,d);v=self.linear(h,p+'v_proj').reshape(b,t,nk,d) + q=self.normalization(q,p+'q_norm');k=self.normalization(k,p+'k_norm');r=c['rope_parameters'];q,k=(rope(z,r['rope_theta'],r['partial_rotary_factor']) for z in (q,k)) + k=np.repeat(k,nh//nk,axis=2);v=np.repeat(v,nh//nk,axis=2) + _,prob=attention(q,k,v) + return {'Q':q,'K':k,'V':v,'P':prob} + p=f'layers.{layer}.linear_attn.';kh=c['linear_num_key_heads'];vh=c['linear_num_value_heads'];kd=c['linear_key_head_dim'];vd=c['linear_value_head_dim'] + x=causal_conv(self.linear(h,p+'in_proj_qkv'),self.small(p+'conv1d.weight')[:,0,:]);q,k,v=np.split(x,[kh*kd,2*kh*kd],axis=-1) + q=q.reshape(b,t,kh,kd);k=k.reshape(b,t,kh,kd);v=v.reshape(b,t,vh,vd) + if vh!=kh:q,k=np.repeat(q,vh//kh,axis=2),np.repeat(k,vh//kh,axis=2) + q=q/np.sqrt(np.sum(q*q,axis=-1,keepdims=True)+1e-6)/np.float32(np.sqrt(kd));k=k/np.sqrt(np.sum(k*k,axis=-1,keepdims=True)+1e-6) + return {'Q':q,'K':k,'V':v} + + def intervene(self,cache,layer,masks): + # Each scenario has a full coherent token-dependent deletion mask. + removed=np.einsum('bth,thd->btd',masks,cache['heads'][0],optimize=True) + mid=cache['raw']+cache['out']-removed + nxt=mid+self.mlp(self.normalization(mid,f'layers.{layer}.post_attention_layernorm'),layer) + return self.recipient(nxt,layer+1) diff --git a/dominance/report.py b/dominance/report.py new file mode 100644 index 0000000..5de7f28 --- /dev/null +++ b/dominance/report.py @@ -0,0 +1,140 @@ +"""Saved-data scientific analysis; no model invocation or fitting.""" +import csv,json +from pathlib import Path +import numpy as np +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +from .core import concentration + + +def stats(x): + x=np.asarray(x,dtype=np.float64);valid=x[np.isfinite(x)] + if not valid.size:return {'count':0,'p10':None,'median':None,'p90':None,'mean':None} + q=np.quantile(valid,[.1,.5,.9]);return {'count':int(valid.size),'p10':float(q[0]),'median':float(q[1]),'p90':float(q[2]),'mean':float(valid.mean())} + + +def csvwrite(p,rows): + with p.open('w') as f: + w=csv.DictWriter(f,fieldnames=list(rows[0]));w.writeheader();w.writerows(rows) + + +def main(root): + root=Path(root);figdir=root/'figures';figdir.mkdir(exist_ok=True);status=json.loads((root/'status.json').read_text());assert status['state']=='completed' + rows=[];joint_rows=[];pool={};curves={};joint_pool={};obs={};winner=[];joint_by_layer={} + for entry in status['completed_links']: + sample=entry['sample'];source=entry['source'];data=dict(np.load(root/'raw'/f'{sample}-link-{source:02d}.npz')) + if 'single_JS' in data: + data['single_sqrtJS']=np.sqrt(data['single_JS']) + for label in ('weak_half','strong_half','random_half','all_heads'): + data['joint_'+label+'_sqrtJS']=np.sqrt(data['joint_'+label+'_JS']) + masks=data.get('mask_'+label,np.ones_like(data['mask_weak_half'])) + data['individual_sum_'+label+'_sqrtJS']=np.sum(data['single_sqrtJS'].transpose(2,1,0)*masks,-1).T + for key in ('Q','K','V','QKV','JS','sqrtJS'): + if 'single_'+key not in data:continue + d=data['single_'+key][:,7:,:].transpose(1,2,0);c=concentration(d);valid=c['valid'];h=d.shape[-1] + metric={k:v.reshape(-1) for k,v in c.items() if k!='valid'} + row={'sample':sample,'source':source,'recipient':source+1,'metric':key,'source_heads':h,'recipient_attention_type':'full_attention' if entry['full_attention_recipient'] else 'linear_attention','observations':d.shape[0]*d.shape[1],'undefined_zero_total':int((~valid).sum())} + for k,v in metric.items(): + pool.setdefault(key,{}).setdefault(k,[]).extend(v) + for s,z in stats(v).items():row[k+'_'+s]=z + rows.append(row) + # Raw per-observation statistics for independently reproducing figures. + for k,v in metric.items():obs.setdefault(key+'_'+k,[]).append(v.astype(np.float32)) + n=d.shape[0]*d.shape[1] + for k,v in [('source',np.full(n,source)),('sample_index',np.full(n,sorted({e['sample'] for e in status['completed_links']}).index(sample))),('token',np.repeat(np.arange(7,32),d.shape[1])),('recipient_head',np.tile(np.arange(d.shape[1]),25))]:obs.setdefault(key+'_'+k,[]).append(v.astype(np.int16)) + p=np.divide(d,d.sum(-1,keepdims=True),out=np.zeros_like(d),where=d.sum(-1,keepdims=True)>0);cum=np.concatenate([np.zeros((*d.shape[:2],1)),np.cumsum(np.sort(p,-1)[...,::-1],-1)],-1) + grid=np.linspace(0,1,101);cc=np.array([np.interp(grid,np.linspace(0,1,h+1),v) for v in cum[valid]]) + curves.setdefault(key,[]).extend(cc) + win=np.argmax(d,-1);counts=np.bincount(win[valid],minlength=h) + winner.append({'sample':sample,'source':source,'metric':key,'most_frequent_winner_fraction':float(counts.max()/max(counts.sum(),1)),'distinct_winning_heads':int((counts>0).sum()),'head_counts':json.dumps(counts.tolist())}) + for label in ('weak_half','strong_half','random_half','all_heads'): + values=data['joint_'+label+'_'+key][7:];summed=data['individual_sum_'+label+'_'+key][7:] + jr={'sample':sample,'source':source,'recipient':source+1,'metric':key,'removed':label} + for k,x in [('joint_effect',values),('sum_individual',summed)]: + for s,z in stats(x).items():jr[k+'_'+s]=z + joint_rows.append(jr);joint_pool.setdefault(key,{}).setdefault(label,[]).extend(values.ravel());joint_by_layer.setdefault((key,label,source),[]).extend(values.ravel()) + csvwrite(root/'concentration-by-link.csv',rows);csvwrite(root/'joint-by-link.csv',joint_rows);csvwrite(root/'winner-identities.csv',winner) + np.savez_compressed(root/'observations.npz',**{k:np.concatenate(v) for k,v in obs.items()}) + pooled={k:{m:stats(v) for m,v in metrics.items()} for k,metrics in pool.items()};joint_stats={k:{m:stats(v) for m,v in metrics.items()} for k,metrics in joint_pool.items()} + paired={} + for key,labels in joint_pool.items(): + weak=np.asarray(labels['weak_half']);strong=np.asarray(labels['strong_half']);random=np.asarray(labels['random_half']);valid=weak>0 + paired[key]={'weak_less_than_strong_fraction':float(np.mean(weak2e-5:raise AssertionError('Null intervention changed recipient') + payload={'single_'+k:v for k,v in singles.items()};payload.update({'joint_'+k:v for k,v in joint.items()});payload.update({'individual_sum_'+k:v for k,v in sum_individual.items()}) + payload.update({'mask_weak_half':weak,'mask_strong_half':strong,'mask_random_half':random}) + payload.update({'null_'+k:v for k,v in null.items()}) + np.savez_compressed(path,**payload) + return {'source':layer,'recipient':layer+1,'source_heads':hs,'recipient_query_heads':hr,'full_attention_recipient':'P' in base,'elapsed_seconds':time.monotonic()-started,'null_max':max(float(np.max(v)) for v in null.values())} + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--source',type=Path,required=True);p.add_argument('--out',type=Path,required=True);p.add_argument('--seconds',type=int,default=7200);a=p.parse_args() + a.out.mkdir(parents=True,exist_ok=False);(a.out/'cache').mkdir();(a.out/'raw').mkdir();started=time.monotonic() + protocol={'scope':'all 23 adjacent layer transitions; source attention head contribution to next effective Q/K/V','samples':'4 previously authored zh/en texts, first32 tokens, descriptive','primary_queries':list(range(7,32)), 'influence':'L2 change / baseline token recipient-head L2; QKV score sqrt(mean of three squared relative changes))','linear_QK':'after causal convolution/SiLU and delta-rule L2 normalization; Q additionally divided by sqrt(key dim)','full_QK':'QK norm and partial RoPE applied; K/V repeated to query heads for indexing (not independent KV observations)','single_ablation':'remove complete source head projected output at all tokens; recompute source postattention normalization, MLP, residual and next input normalization','joint_ablation':'weak/strong/random half and all attention heads; one coherent token-varying removal mask for each recipient query head; ranking uses QKV single effects; no fitting','continuous':'effective heads=1/sum(p^2); fraction=effective/H; concentration=(sum(p^2)-1/H)/(1-1/H)','zero_total':'undefined concentration (NaN arrays), count explicitly','cost_budget_seconds':a.seconds,'no_training':True,'keep_all_weights_and_caches':True,'limitations':['not final LM quality or pruning proof','individual effects not additive; normalized scores not true contribution probabilities','token-varying masks are input-specific diagnostic and not a deployable fixed head subset','linear attention does not have an ordinary softmax probability matrix','Q/K/V scales handled separately; combined QKV is a declared balanced relative-effect score']} + dump(a.out/'protocol.json',protocol);status={'state':'running','completed_links':[]};dump(a.out/'status.json',status) + try: + lm=InfluenceLM(a.source/'weights',json.loads((a.source/'model_config.json').read_text()),seconds=a.seconds) + samples=json.loads(Path('crossqkv/fresh_samples.json').read_text())['samples'];tok=Tokenizer.from_file(str(a.source/'tokenizer.json'));controls=[] + for si,s in enumerate(samples): + lm.guard();s['token_ids']=tok.encode(s['text']).ids[:32];ids=np.array([s['token_ids']],dtype=np.int64);cache=a.out/'cache'/s['id'];cache.mkdir() + final=lm.baseline(ids,cache);np.save(cache/'final.npy',final) + for x in lm.head_sum_errors:controls.append({'sample':s['id'],**x}) + if si==0: + reference,_=StreamingLM.forward(lm,ids);normalized=lm.normalization(final,'norm');err=float(np.linalg.norm(normalized-reference)/max(np.linalg.norm(reference),1e-30));assert err<2e-5;status['canonical_forward_relative_error']=err + dump(a.out/'head-sum-controls.json',controls);dump(a.out/'samples.json',samples) + for layer in range(23): + target=a.out/'raw'/f'{s["id"]}-link-{layer:02d}.npz' + with np.load(cache/f'layer-{layer}.npz') as data:loaded={k:data[k] for k in data.files} + info=link(lm,loaded,layer,target,31000+si*100+layer) + del loaded + status['completed_links'].append({'sample':s['id'],**info});status['elapsed_seconds']=time.monotonic()-started;dump(a.out/'status.json',status) + print('completed',s['id'],layer,'seconds',round(info['elapsed_seconds'],2),flush=True) + status.update(state='completed',elapsed_seconds=time.monotonic()-started,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss);dump(a.out/'status.json',status) + except BaseException as e: + status.update(state='failed',error=repr(e),elapsed_seconds=time.monotonic()-started);dump(a.out/'status.json',status);raise +if __name__=='__main__':main() diff --git a/results/head-dominance/REPORT.md b/results/head-dominance/REPORT.md new file mode 100644 index 0000000..063af5a --- /dev/null +++ b/results/head-dominance/REPORT.md @@ -0,0 +1,48 @@ +# 上游注意力头对下一层 Q/K/V 的影响集中度 + +实测全部23个相邻层连接、4条既有中英文文本各32 token。主统计取query7–31,按文本/token/接收query头分别计算,再汇总。不是独立样本的显著性检验。 + +影响是移除上游一个头后,下一层Q/K/V向量的相对L2变化;重新计算中间MLP/残差/归一化。各头影响归一化仅构成消融影响分布,不是天然可相加的贡献概率。 + +| 对象 | 有效头比例中位数 [P10,P90] | 最大头份额中位数 | 前25%头份额中位数 | 最小25%头份额中位数 | 90%影响所需头数中位数 | +|---|---:|---:|---:|---:|---:| +| Q | 0.736 [0.506,0.861] | 18.6% | 46.0% | 10.0% | 12.0 | +| K | 0.749 [0.517,0.879] | 18.4% | 45.2% | 10.3% | 12.0 | +| V | 0.723 [0.479,0.862] | 19.0% | 46.6% | 9.5% | 12.0 | +| QKV | 0.760 [0.531,0.878] | 18.1% | 44.6% | 10.6% | 12.0 | +| JS | 0.244 [0.117,0.434] | 42.3% | 82.3% | 1.1% | 6.0 | +| sqrtJS | 0.558 [0.362,0.730] | 21.7% | 56.5% | 6.8% | 11.0 | + +sqrtJS为JS距离尺度敏感性分析:在初次完整汇总后增加,原始消融和原JS主结果不变。JS散度近似平方尺度,归一化后会放大强弱差距;JS影响份额不是注意力概率质量份额。 + +按接收层类型分层(避免总体分布混合不同机制): + +| 接收层类型 | Q有效头比例中位数 | K | V | QKV | +|---|---:|---:|---:|---:| +| linear_attention | 0.747 | 0.773 | 0.736 | 0.778 | +| full_attention | 0.677 | 0.562 | 0.629 | 0.645 | + +QKV是三个相对变化平方的均值再开根号,避免混加不同Q/K/V尺度;JS仅普通注意力接收层有。普通注意力GQA的K/V复制到query头索引用于配对,不是额外独立KV头。 + +| 联合移除 | QKV相对影响中位数 [P10,P90] | 检索JS中位数 [P10,P90] | +|---|---:|---:| +| weak_half | 0.09035 [0.03148,0.23354] | 0.001413 [0.000220,0.006160] | +| strong_half | 0.20918 [0.09105,0.48031] | 0.025745 [0.004460,0.088616] | +| random_half | 0.16420 [0.06186,0.39711] | 0.013815 [0.002647,0.055241] | +| all_heads | 0.22908 [0.09930,0.51739] | 0.028837 [0.004936,0.100283] | + +配对观测:QKV弱半数的实际联合影响小于强半数的比例为95.6%;强/弱影响比中位数为2.358。这是逐观测比值的中位数,不是两个总体中位数之比。弱效应为零的观测另计,不做除法。 + +联合实验的弱/强/随机半数头,在各token、各接收头上分别选择,形成完整序列再重算。排名依据单头QKV影响,同样用于JS;它是输入自适应诊断,不是免费可用的剪枝策略。线性注意力的卷积和普通注意力的QK可能传播跨token影响。所有上游头移除仍保留残差背景,不能把它称为删除全部上游信息。 + +连续指标:N_eff=1/Σp²;有效头比例=N_eff/H;C=(Σp²−1/H)/(1−1/H)。全零总影响时集中度未定义,原始数组保留NaN并在统计时排除、另计数量。单头相对L2影响在做头间占比归一化之前的总和及最大值均保留,避免把微小效应中的集中误认作重要。字段max_absolute指最大无符号相对效应,并非隐藏坐标中的未归一化L2;total是单头效应之和,真正全头联合效果另测。 + +联合实验中保存的单头影响之和仅为按query掩码加权的参考。单头消融移除整段序列的一个头,联合掩码则可跨位置变化;二者的差不能单独解释成纯非线性交互,因为时间支持也不同。应直接比较实际弱/强/随机联合效果。 + +## 文件与限制 + +`raw/*.npz`保留每次单头影响、联合影响、相应单头影响之和、删除掩码及空操作对照;`observations.npz`保留逐观测连续指标及索引;CSV和JSON为全部图表源表。原模型权重和大基线缓存继续保留在服务器。 + +只测短文本上的局部下一层作用;不代表全LM质量、可直接删头、所有输入普遍集中,或相似头可互相替代。所有头贡献可能是广泛分布的;报告不以发现稀疏为成功条件。 + +CPU测量耗时2636.4秒,峰值RSS155.9 MiB;无训练、GPU或清理。 diff --git a/results/head-dominance/analysis-summary.json b/results/head-dominance/analysis-summary.json new file mode 100644 index 0000000..d724d24 --- /dev/null +++ b/results/head-dominance/analysis-summary.json @@ -0,0 +1,1490 @@ +{ + "scale_sensitivity": "sqrt(JS) added after reviewing completed primary summaries; primary JS and all raw interventions unchanged; squared divergences cannot be equated to probability mass or linear-scale effects", + "by_recipient_type": { + "Q": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.42808762192726135, + "median": 0.8017966151237488, + "p90": 1.2796676754951481, + "mean": 0.828383005860572 + }, + "max_absolute": { + "count": 4800, + "p10": 0.07022316381335258, + "median": 0.13597501814365387, + "p90": 0.29393495619297033, + "mean": 0.1614389501255937 + }, + "effective_heads": { + "count": 4800, + "p10": 7.634387731552124, + "median": 10.829116821289062, + "p90": 12.903363132476807, + "mean": 10.545973850339651 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.47714923322200775, + "median": 0.6768198013305664, + "p90": 0.8064601957798004, + "mean": 0.6591233656462282 + }, + "concentration": { + "count": 4800, + "p10": 0.015999119356274606, + "median": 0.031833209097385406, + "p90": 0.07305203005671501, + "mean": 0.038962758334334165 + }, + "top1": { + "count": 4800, + "p10": 0.13040318638086318, + "median": 0.1768590658903122, + "p90": 0.27592835426330575, + "mean": 0.1917892166459933 + }, + "top25": { + "count": 4800, + "p10": 0.4181676685810089, + "median": 0.49224844574928284, + "p90": 0.5894871711730958, + "mean": 0.4984907660695414 + }, + "top50": { + "count": 4800, + "p10": 0.6820279717445373, + "median": 0.7461760938167572, + "p90": 0.8289147377014161, + "mean": 0.7511068653936187 + }, + "bottom25": { + "count": 4800, + "p10": 0.04171919822692871, + "median": 0.0870458073914051, + "p90": 0.12668813914060592, + "mean": 0.0858515575454415 + }, + "heads90": { + "count": 4800, + "p10": 10.0, + "median": 12.0, + "p90": 13.0, + "mean": 11.924375 + } + }, + "linear_attention": { + "total": { + "count": 27200, + "p10": 0.1771158680319786, + "median": 0.7782308757305145, + "p90": 1.8638553977012642, + "mean": 0.9841912085086326 + }, + "max_absolute": { + "count": 27200, + "p10": 0.027340889908373356, + "median": 0.13787434250116348, + "p90": 0.4472817718982699, + "mean": 0.2126326917772873 + }, + "effective_heads": { + "count": 27200, + "p10": 5.968966674804688, + "median": 10.08900499343872, + "p90": 13.287356853485107, + "mean": 9.637892792207353 + }, + "effective_fraction": { + "count": 27200, + "p10": 0.5126844048500061, + "median": 0.7466321587562561, + "p90": 0.8671843826770783, + "mean": 0.7170763921009048 + }, + "concentration": { + "count": 27200, + "p10": 0.012945951707661153, + "median": 0.02890323381870985, + "p90": 0.07301022931933407, + "mean": 0.03798074354623489 + }, + "top1": { + "count": 27200, + "p10": 0.12049156576395034, + "median": 0.18831469118595123, + "p90": 0.3018690258264542, + "mean": 0.20151331216672105 + }, + "top25": { + "count": 27200, + "p10": 0.38046165108680724, + "median": 0.453884556889534, + "p90": 0.5794604063034058, + "mean": 0.4684076986615272 + }, + "top50": { + "count": 27200, + "p10": 0.650843471288681, + "median": 0.7193152010440826, + "p90": 0.8102161049842835, + "mean": 0.7255355768234414 + }, + "bottom25": { + "count": 27200, + "p10": 0.05939290076494217, + "median": 0.10252666845917702, + "p90": 0.14262291491031648, + "mean": 0.1017356713594499 + }, + "heads90": { + "count": 27200, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.579007352941176 + } + } + }, + "K": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.2932938843965531, + "median": 0.751684695482254, + "p90": 1.2453256130218517, + "mean": 0.7491105496883392 + }, + "max_absolute": { + "count": 4800, + "p10": 0.05981316082179546, + "median": 0.16251461952924728, + "p90": 0.29028533995151523, + "mean": 0.17104425797394165 + }, + "effective_heads": { + "count": 4800, + "p10": 6.421846866607666, + "median": 8.99651575088501, + "p90": 11.717314338684085, + "mean": 9.006867933471998 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.4013654291629791, + "median": 0.5622822344303131, + "p90": 0.7323321461677553, + "mean": 0.5629292458419999 + }, + "concentration": { + "count": 4800, + "p10": 0.024366708286106587, + "median": 0.05189775675535202, + "p90": 0.09943302348256117, + "mean": 0.05819399481251215 + }, + "top1": { + "count": 4800, + "p10": 0.15217750519514084, + "median": 0.22362081706523895, + "p90": 0.333113169670105, + "mean": 0.2327083595407506 + }, + "top25": { + "count": 4800, + "p10": 0.46408111453056333, + "median": 0.554540365934372, + "p90": 0.6376730680465699, + "mean": 0.5528365189085404 + }, + "top50": { + "count": 4800, + "p10": 0.7256738007068634, + "median": 0.7895856201648712, + "p90": 0.8492424726486212, + "mean": 0.7887261419494946 + }, + "bottom25": { + "count": 4800, + "p10": 0.037170265614986424, + "median": 0.06792030483484268, + "p90": 0.10355381369590763, + "mean": 0.06925618241191842 + }, + "heads90": { + "count": 4800, + "p10": 10.0, + "median": 11.0, + "p90": 13.0, + "mean": 11.344166666666666 + } + }, + "linear_attention": { + "total": { + "count": 27200, + "p10": 0.27437427937984465, + "median": 0.7400951087474823, + "p90": 1.6867972612380997, + "mean": 0.9442492292869343 + }, + "max_absolute": { + "count": 27200, + "p10": 0.04476405493915081, + "median": 0.12380896881222725, + "p90": 0.3700851738452914, + "mean": 0.18057506222894196 + }, + "effective_heads": { + "count": 27200, + "p10": 6.289550733566284, + "median": 10.748710632324219, + "p90": 13.59198980331421, + "mean": 10.086356610287638 + }, + "effective_fraction": { + "count": 27200, + "p10": 0.567463606595993, + "median": 0.7731030285358429, + "p90": 0.8852756083011627, + "mean": 0.7487537822614917 + }, + "concentration": { + "count": 27200, + "p10": 0.01110830344259739, + "median": 0.024465462192893028, + "p90": 0.059676705300807986, + "mean": 0.03137563373759108 + }, + "top1": { + "count": 27200, + "p10": 0.11619977951049805, + "median": 0.17642024159431458, + "p90": 0.2702268481254578, + "mean": 0.18697812774930805 + }, + "top25": { + "count": 27200, + "p10": 0.3685951292514801, + "median": 0.4383372962474823, + "p90": 0.5479569613933564, + "mean": 0.44980560849694645 + }, + "top50": { + "count": 27200, + "p10": 0.6412311494350433, + "median": 0.7067530751228333, + "p90": 0.7897655248641968, + "mean": 0.7119768214488731 + }, + "bottom25": { + "count": 27200, + "p10": 0.06756735369563102, + "median": 0.10880804806947708, + "p90": 0.14901476353406906, + "mean": 0.10839668871242526 + }, + "heads90": { + "count": 27200, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.763308823529412 + } + } + }, + "V": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.2459064647555352, + "median": 0.7926892042160034, + "p90": 1.4473202824592595, + "mean": 0.8259200246507923 + }, + "max_absolute": { + "count": 4800, + "p10": 0.04807887561619282, + "median": 0.14048674702644348, + "p90": 0.3213535517454149, + "mean": 0.1687076044936354 + }, + "effective_heads": { + "count": 4800, + "p10": 6.27513017654419, + "median": 10.059541702270508, + "p90": 13.006668281555179, + "mean": 9.797848190466564 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.39219563603401186, + "median": 0.6287213563919067, + "p90": 0.8129167675971987, + "mean": 0.6123655119041602 + }, + "concentration": { + "count": 4800, + "p10": 0.01534255063161254, + "median": 0.039368655532598495, + "p90": 0.1033165372908116, + "mean": 0.054241599862386164 + }, + "top1": { + "count": 4800, + "p10": 0.12827315628528596, + "median": 0.1957768052816391, + "p90": 0.32790936529636394, + "mean": 0.21723997221017877 + }, + "top25": { + "count": 4800, + "p10": 0.40897682309150696, + "median": 0.5191850960254669, + "p90": 0.6538566529750826, + "mean": 0.5266173542042574 + }, + "top50": { + "count": 4800, + "p10": 0.6763901829719543, + "median": 0.7685472369194031, + "p90": 0.8522575080394748, + "mean": 0.7653420636554559 + }, + "bottom25": { + "count": 4800, + "p10": 0.038462628796696666, + "median": 0.07527097687125206, + "p90": 0.1310943305492402, + "mean": 0.08093525390451153 + }, + "heads90": { + "count": 4800, + "p10": 10.0, + "median": 12.0, + "p90": 13.0, + "mean": 11.665 + } + }, + "linear_attention": { + "total": { + "count": 27200, + "p10": 0.29220492839813234, + "median": 0.5876882374286652, + "p90": 1.7076982617378238, + "mean": 0.8579155112553717 + }, + "max_absolute": { + "count": 27200, + "p10": 0.04601397477090359, + "median": 0.11557785794138908, + "p90": 0.3890607744455338, + "mean": 0.18123242851627497 + }, + "effective_heads": { + "count": 27200, + "p10": 5.704750633239746, + "median": 9.907225608825684, + "p90": 13.273716259002686, + "mean": 9.52269080659046 + }, + "effective_fraction": { + "count": 27200, + "p10": 0.5001892507076263, + "median": 0.736346960067749, + "p90": 0.8675740957260132, + "mean": 0.7092839469459347 + }, + "concentration": { + "count": 27200, + "p10": 0.012901593837887049, + "median": 0.029933489859104156, + "p90": 0.07959845960140231, + "mean": 0.03960282899909276 + }, + "top1": { + "count": 27200, + "p10": 0.12066357731819152, + "median": 0.1884898841381073, + "p90": 0.30654583871364594, + "mean": 0.20284635099856294 + }, + "top25": { + "count": 27200, + "p10": 0.38022677004337313, + "median": 0.4595302492380142, + "p90": 0.5857594251632692, + "mean": 0.47216263436109707 + }, + "top50": { + "count": 27200, + "p10": 0.6528049886226654, + "median": 0.7263502180576324, + "p90": 0.8170590281486512, + "mean": 0.7314765423689695 + }, + "bottom25": { + "count": 27200, + "p10": 0.055867613106966016, + "median": 0.09783932939171791, + "p90": 0.14103657454252244, + "mean": 0.09802515388039276 + }, + "heads90": { + "count": 27200, + "p10": 6.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.472205882352942 + } + } + }, + "QKV": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.3599545568227768, + "median": 0.8310977220535278, + "p90": 1.3057362198829652, + "mean": 0.8320240748580545 + }, + "max_absolute": { + "count": 4800, + "p10": 0.06178740113973618, + "median": 0.1471318155527115, + "p90": 0.2991165637969971, + "mean": 0.16522609861179566 + }, + "effective_heads": { + "count": 4800, + "p10": 7.475884056091308, + "median": 10.319912433624268, + "p90": 12.712833499908447, + "mean": 10.20285238613685 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.46724275350570676, + "median": 0.6449945271015167, + "p90": 0.7945520937442779, + "mean": 0.6376782741335532 + }, + "concentration": { + "count": 4800, + "p10": 0.017238049022853374, + "median": 0.03669338673353195, + "p90": 0.07601434364914894, + "mean": 0.04257073526978881 + }, + "top1": { + "count": 4800, + "p10": 0.13321341276168824, + "median": 0.18747321516275406, + "p90": 0.28328865170478823, + "mean": 0.1998321647165964 + }, + "top25": { + "count": 4800, + "p10": 0.4247431606054306, + "median": 0.5103605389595032, + "p90": 0.5986294209957124, + "mean": 0.5111861900550624 + }, + "top50": { + "count": 4800, + "p10": 0.6898793756961823, + "median": 0.7608179152011871, + "p90": 0.826508241891861, + "mean": 0.759507774884502 + }, + "bottom25": { + "count": 4800, + "p10": 0.043933799117803575, + "median": 0.0804036296904087, + "p90": 0.12289210483431817, + "mean": 0.08222739746629183 + }, + "heads90": { + "count": 4800, + "p10": 10.0, + "median": 12.0, + "p90": 13.0, + "mean": 11.84 + } + }, + "linear_attention": { + "total": { + "count": 27200, + "p10": 0.300345253944397, + "median": 0.780917763710022, + "p90": 1.794381380081177, + "mean": 0.9848778233173139 + }, + "max_absolute": { + "count": 27200, + "p10": 0.04775628037750721, + "median": 0.12801389396190643, + "p90": 0.40935705900192265, + "mean": 0.19344240318685907 + }, + "effective_heads": { + "count": 27200, + "p10": 6.223408126831055, + "median": 10.616299629211426, + "p90": 13.700624370574952, + "mean": 10.054770943066654 + }, + "effective_fraction": { + "count": 27200, + "p10": 0.551340651512146, + "median": 0.7776066660881042, + "p90": 0.8834639668464661, + "mean": 0.7460366055385812 + }, + "concentration": { + "count": 27200, + "p10": 0.010811386536806822, + "median": 0.02463168278336525, + "p90": 0.0634766072034836, + "mean": 0.032149144387093494 + }, + "top1": { + "count": 27200, + "p10": 0.11426723748445511, + "median": 0.17895939201116562, + "p90": 0.2811527192592621, + "mean": 0.18969931307632257 + }, + "top25": { + "count": 27200, + "p10": 0.3703198492527008, + "median": 0.43639612197875977, + "p90": 0.5559773981571198, + "mean": 0.45095220466199165 + }, + "top50": { + "count": 27200, + "p10": 0.6413009822368622, + "median": 0.7052080035209656, + "p90": 0.7924475073814392, + "mean": 0.7119353248003651 + }, + "bottom25": { + "count": 27200, + "p10": 0.06760622188448906, + "median": 0.11000294238328934, + "p90": 0.1492564007639885, + "mean": 0.10894173061254311 + }, + "heads90": { + "count": 27200, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.776801470588236 + } + } + }, + "JS": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.006019792705774307, + "median": 0.02659580297768116, + "p90": 0.07109484225511555, + "mean": 0.03422935250632387 + }, + "max_absolute": { + "count": 4800, + "p10": 0.0024493230041116474, + "median": 0.009726522956043482, + "p90": 0.03804903179407121, + "mean": 0.01699844059891954 + }, + "effective_heads": { + "count": 4800, + "p10": 1.8775771021842957, + "median": 3.901782989501953, + "p90": 6.948997354507451, + "mean": 4.195899176249902 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.11734856888651848, + "median": 0.24386143684387207, + "p90": 0.4343123346567157, + "mean": 0.26224369851561885 + }, + "concentration": { + "count": 4800, + "p10": 0.08683274388313295, + "median": 0.20671262592077255, + "p90": 0.5014413654804231, + "mean": 0.25655510467089093 + }, + "top1": { + "count": 4800, + "p10": 0.24280447959899903, + "median": 0.4230606108903885, + "p90": 0.7145513713359836, + "mean": 0.45466951202290756 + }, + "top25": { + "count": 4800, + "p10": 0.6646065950393677, + "median": 0.8229818344116211, + "p90": 0.9326550900936127, + "mean": 0.8089195707378288 + }, + "top50": { + "count": 4800, + "p10": 0.8803883492946625, + "median": 0.9476927518844604, + "p90": 0.9820005834102631, + "mean": 0.9382064804434777 + }, + "bottom25": { + "count": 4800, + "p10": 0.003459757659584284, + "median": 0.011073274537920952, + "p90": 0.03202965445816518, + "mean": 0.014963302712030781 + }, + "heads90": { + "count": 4800, + "p10": 3.0, + "median": 6.0, + "p90": 9.0, + "mean": 6.310208333333334 + } + } + }, + "sqrtJS": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.22150191217660903, + "median": 0.4924028515815735, + "p90": 0.7786339044570924, + "mean": 0.493503825609417 + }, + "max_absolute": { + "count": 4800, + "p10": 0.04949063360691071, + "median": 0.0986231379210949, + "p90": 0.195061606168747, + "mean": 0.11290162225719541 + }, + "effective_heads": { + "count": 4800, + "p10": 5.784081172943115, + "median": 8.931822299957275, + "p90": 11.672658061981203, + "mean": 8.82100272881488 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.3615050733089447, + "median": 0.5582388937473297, + "p90": 0.7295411288738252, + "mean": 0.55131267055093 + }, + "concentration": { + "count": 4800, + "p10": 0.02471497356891632, + "median": 0.052756521850824356, + "p90": 0.11774752512574196, + "mean": 0.06432512730980913 + }, + "top1": { + "count": 4800, + "p10": 0.14574055671691893, + "median": 0.21746353060007095, + "p90": 0.345387390255928, + "mean": 0.23503983538908263 + }, + "top25": { + "count": 4800, + "p10": 0.4643830180168152, + "median": 0.5649686753749847, + "p90": 0.6741052329540254, + "mean": 0.5682829620999594 + }, + "top50": { + "count": 4800, + "p10": 0.7257072806358338, + "median": 0.7975856065750122, + "p90": 0.8583120703697206, + "mean": 0.794721678632001 + }, + "bottom25": { + "count": 4800, + "p10": 0.04389082230627537, + "median": 0.06803373247385025, + "p90": 0.10318452790379527, + "mean": 0.07107512763216316 + }, + "heads90": { + "count": 4800, + "p10": 10.0, + "median": 11.0, + "p90": 13.0, + "mean": 11.273125 + } + } + } + }, + "full_attention_source_indices": [ + 2, + 6, + 10, + 14, + 18, + 22 + ], + "paired_joint": { + "Q": { + "weak_less_than_strong_fraction": 0.9558125, + "weak_less_than_random_fraction": 0.92478125, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 32000, + "p10": 1.2414084553718567, + "median": 2.406548857688904, + "p90": 4.8988345146179215, + "mean": 2.8476133695249444 + }, + "random_over_weak": { + "count": 32000, + "p10": 1.0663463115692138, + "median": 1.7550507187843323, + "p90": 3.4048652887344364, + "mean": 2.0716914434591307 + } + }, + "K": { + "weak_less_than_strong_fraction": 0.94053125, + "weak_less_than_random_fraction": 0.93659375, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 32000, + "p10": 1.1599948525428772, + "median": 2.2635676860809326, + "p90": 4.8068671226501465, + "mean": 2.721669250183739 + }, + "random_over_weak": { + "count": 32000, + "p10": 1.0976211547851562, + "median": 1.7731841802597046, + "p90": 3.4223729372024536, + "mean": 2.0756989874048157 + } + }, + "V": { + "weak_less_than_strong_fraction": 0.9139375, + "weak_less_than_random_fraction": 0.89215625, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 32000, + "p10": 1.0545674324035645, + "median": 2.339610457420349, + "p90": 5.230419206619263, + "mean": 2.8710654100831596 + }, + "random_over_weak": { + "count": 32000, + "p10": 0.9815070390701294, + "median": 1.7466103434562683, + "p90": 3.7226275205612196, + "mean": 2.160556613523513 + } + }, + "QKV": { + "weak_less_than_strong_fraction": 0.95575, + "weak_less_than_random_fraction": 0.9521875, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 32000, + "p10": 1.2344148278236389, + "median": 2.357925534248352, + "p90": 4.5909470558166525, + "mean": 2.7040380254182965 + }, + "random_over_weak": { + "count": 32000, + "p10": 1.1472956657409668, + "median": 1.7747376561164856, + "p90": 3.2936715126037615, + "mean": 2.046528765618801 + } + }, + "JS": { + "weak_less_than_strong_fraction": 0.9970833333333333, + "weak_less_than_random_fraction": 0.988125, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 4800, + "p10": 4.6294151782989506, + "median": 18.771913528442383, + "p90": 68.13132858276367, + "mean": 32.662432227910806 + }, + "random_over_weak": { + "count": 4800, + "p10": 2.710283899307251, + "median": 10.478245258331299, + "p90": 39.4547286987305, + "mean": 19.306766499467194 + } + }, + "sqrtJS": { + "weak_less_than_strong_fraction": 0.9970833333333333, + "weak_less_than_random_fraction": 0.988125, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 4800, + "p10": 2.1516075849533083, + "median": 4.332656145095825, + "p90": 8.254170131683349, + "mean": 4.931455753048261 + }, + "random_over_weak": { + "count": 4800, + "p10": 1.6462939023971557, + "median": 3.2370117902755737, + "p90": 6.281299448013308, + "mean": 3.7469815428182485 + } + } + }, + "pooling": "query7..31, per text/token/recipient-query-head; descriptive dependent observations, not independent samples", + "concentration": { + "Q": { + "total": { + "count": 32000, + "p10": 0.19806515499949456, + "median": 0.7841357989236712, + "p90": 1.727399396291003, + "mean": 0.9608199778404878 + }, + "max_absolute": { + "count": 32000, + "p10": 0.03213169947266579, + "median": 0.13753367960453033, + "p90": 0.40950461924076104, + "mean": 0.20495363052953325 + }, + "effective_heads": { + "count": 32000, + "p10": 6.074677358285013, + "median": 10.278813612228042, + "p90": 13.231621453644234, + "mean": 9.774104951134538 + }, + "effective_fraction": { + "count": 32000, + "p10": 0.5059540464573292, + "median": 0.736010784821391, + "p90": 0.8612895367324331, + "mean": 0.7083834381455979 + }, + "concentration": { + "count": 32000, + "p10": 0.01331159273601288, + "median": 0.029374472335226654, + "p90": 0.07302650438478069, + "mean": 0.03812804576861353 + }, + "top1": { + "count": 32000, + "p10": 0.12190424615545024, + "median": 0.1862308612148443, + "p90": 0.2983967835528167, + "mean": 0.20005469782669785 + }, + "top25": { + "count": 32000, + "p10": 0.38400657936407756, + "median": 0.45974826740579305, + "p90": 0.5813789312182575, + "mean": 0.4729201587141697 + }, + "top50": { + "count": 32000, + "p10": 0.6545684499580247, + "median": 0.7233334799076764, + "p90": 0.8135213374924113, + "mean": 0.7293712701244147 + }, + "bottom25": { + "count": 32000, + "p10": 0.056496162230507345, + "median": 0.10028684220891418, + "p90": 0.14090863982385138, + "mean": 0.09935305428362357 + }, + "heads90": { + "count": 32000, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.7808125 + } + }, + "K": { + "total": { + "count": 32000, + "p10": 0.27868743662256745, + "median": 0.7418987276032567, + "p90": 1.5808700304012746, + "mean": 0.9149784272735005 + }, + "max_absolute": { + "count": 32000, + "p10": 0.04672095738351345, + "median": 0.12887775897979736, + "p90": 0.34267061948776245, + "mean": 0.17914544159069193 + }, + "effective_heads": { + "count": 32000, + "p10": 6.2997924969825645, + "median": 10.282293972258437, + "p90": 13.459263973201868, + "mean": 9.924433305753986 + }, + "effective_fraction": { + "count": 32000, + "p10": 0.5169652544140182, + "median": 0.7494478491534898, + "p90": 0.8792124936928435, + "mean": 0.7208801016057295 + }, + "concentration": { + "count": 32000, + "p10": 0.011776948469577102, + "median": 0.027261487005195033, + "p90": 0.0685273479698932, + "mean": 0.03539838788532774 + }, + "top1": { + "count": 32000, + "p10": 0.11868254832556942, + "median": 0.1837866249324059, + "p90": 0.2811234081621101, + "mean": 0.19383766248805132 + }, + "top25": { + "count": 32000, + "p10": 0.37291786446716907, + "median": 0.4519532440455542, + "p90": 0.5801074854843632, + "mean": 0.4652602449275946 + }, + "top50": { + "count": 32000, + "p10": 0.6453049612946737, + "median": 0.7175863432426386, + "p90": 0.8124186772934331, + "mean": 0.7234892194956966 + }, + "bottom25": { + "count": 32000, + "p10": 0.05735948077175399, + "median": 0.1032195714581893, + "p90": 0.14622727669722033, + "mean": 0.10252561275154358 + }, + "heads90": { + "count": 32000, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.8504375 + } + }, + "V": { + "total": { + "count": 32000, + "p10": 0.2812398451380432, + "median": 0.6038189655519091, + "p90": 1.645845576003194, + "mean": 0.8531161886132836 + }, + "max_absolute": { + "count": 32000, + "p10": 0.04643451571464539, + "median": 0.1182936355471611, + "p90": 0.3698546379804613, + "mean": 0.17935370491287903 + }, + "effective_heads": { + "count": 32000, + "p10": 5.736491743713151, + "median": 9.942905498299558, + "p90": 13.239749202567198, + "mean": 9.563964412704575 + }, + "effective_fraction": { + "count": 32000, + "p10": 0.47907725223804926, + "median": 0.7227435911028347, + "p90": 0.8621985150352091, + "mean": 0.6947461816057294 + }, + "concentration": { + "count": 32000, + "p10": 0.013188755715480607, + "median": 0.03110799980364811, + "p90": 0.08225759989950268, + "mean": 0.0417986446199604 + }, + "top1": { + "count": 32000, + "p10": 0.12161388449889787, + "median": 0.1896916578538292, + "p90": 0.30860335061787136, + "mean": 0.20500539421837363 + }, + "top25": { + "count": 32000, + "p10": 0.3836189886588612, + "median": 0.4664460271280716, + "p90": 0.5998502376306984, + "mean": 0.4803308423023778 + }, + "top50": { + "count": 32000, + "p10": 0.6551911500035564, + "median": 0.7314303909565261, + "p90": 0.8252102923927693, + "mean": 0.736556370556187 + }, + "bottom25": { + "count": 32000, + "p10": 0.05101045030368696, + "median": 0.09549013543061644, + "p90": 0.13985548819761448, + "mean": 0.09546166889828221 + }, + "heads90": { + "count": 32000, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.651125 + } + }, + "QKV": { + "total": { + "count": 32000, + "p10": 0.31270650094375013, + "median": 0.7898262748494744, + "p90": 1.689432829990983, + "mean": 0.96194976092564 + }, + "max_absolute": { + "count": 32000, + "p10": 0.0506718147546053, + "median": 0.13108564168214798, + "p90": 0.375281447172165, + "mean": 0.18920995750059955 + }, + "effective_heads": { + "count": 32000, + "p10": 6.304201972468572, + "median": 10.534389403054181, + "p90": 13.605718976137743, + "mean": 10.076983159324206 + }, + "effective_fraction": { + "count": 32000, + "p10": 0.531184131354781, + "median": 0.7600632811713559, + "p90": 0.8777116494832276, + "mean": 0.7297828558447914 + }, + "concentration": { + "count": 32000, + "p10": 0.01127297981394185, + "median": 0.026350765843306763, + "p90": 0.06576838727607895, + "mean": 0.03371238303288477 + }, + "top1": { + "count": 32000, + "p10": 0.11599946413523492, + "median": 0.1805094629907935, + "p90": 0.2813719110798307, + "mean": 0.19121924080514013 + }, + "top25": { + "count": 32000, + "p10": 0.37423556131777447, + "median": 0.446468832512783, + "p90": 0.5689850647038045, + "mean": 0.4599873026303993 + }, + "top50": { + "count": 32000, + "p10": 0.6450496968093983, + "median": 0.71281259036529, + "p90": 0.8019362921144723, + "mean": 0.7190711923720653 + }, + "bottom25": { + "count": 32000, + "p10": 0.06218593053663734, + "median": 0.10599167602648567, + "p90": 0.14689275149483338, + "mean": 0.10493458066537943 + }, + "heads90": { + "count": 32000, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.93628125 + } + }, + "JS": { + "total": { + "count": 4800, + "p10": 0.006019792613687969, + "median": 0.026595803305099253, + "p90": 0.07109484240791066, + "mean": 0.03422935252324174 + }, + "max_absolute": { + "count": 4800, + "p10": 0.0024493230041116474, + "median": 0.009726522956043482, + "p90": 0.03804903179407121, + "mean": 0.01699844059891954 + }, + "effective_heads": { + "count": 4800, + "p10": 1.8775771328772404, + "median": 3.901782998659897, + "p90": 6.9489972185767925, + "mean": 4.1958991748153185 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.11734857080482752, + "median": 0.24386143741624355, + "p90": 0.43431232616104953, + "mean": 0.2622436984259574 + }, + "concentration": { + "count": 4800, + "p10": 0.08683274490451195, + "median": 0.20671262373235755, + "p90": 0.5014413860763198, + "mean": 0.25655510469396303 + }, + "top1": { + "count": 4800, + "p10": 0.24280447987972617, + "median": 0.4230606060703387, + "p90": 0.7145513508216859, + "mean": 0.45466951196884586 + }, + "top25": { + "count": 4800, + "p10": 0.6646066107988152, + "median": 0.8229818456850793, + "p90": 0.9326550608746287, + "mean": 0.808919570864033 + }, + "top50": { + "count": 4800, + "p10": 0.8803883543181658, + "median": 0.9476927560262463, + "p90": 0.9820005650641489, + "mean": 0.9382064806256976 + }, + "bottom25": { + "count": 4800, + "p10": 0.0034597576965423498, + "median": 0.01107327468889098, + "p90": 0.03202965476590303, + "mean": 0.014963302718693967 + }, + "heads90": { + "count": 4800, + "p10": 3.0, + "median": 6.0, + "p90": 9.0, + "mean": 6.310208333333334 + } + }, + "sqrtJS": { + "total": { + "count": 4800, + "p10": 0.22150190912652762, + "median": 0.4924028478562832, + "p90": 0.7786339074373245, + "mean": 0.4935038255766646 + }, + "max_absolute": { + "count": 4800, + "p10": 0.04949063360691071, + "median": 0.0986231379210949, + "p90": 0.195061606168747, + "mean": 0.11290162225719541 + }, + "effective_heads": { + "count": 4800, + "p10": 5.7840811612328125, + "median": 8.93182211792415, + "p90": 11.67265834734814, + "mean": 8.821002725510846 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.3615050725770508, + "median": 0.5582388823702594, + "p90": 0.7295411467092587, + "mean": 0.5513126703444279 + }, + "concentration": { + "count": 4800, + "p10": 0.02471497387658298, + "median": 0.05275652115976448, + "p90": 0.11774752758475547, + "mean": 0.06432512729180001 + }, + "top1": { + "count": 4800, + "p10": 0.1457405612874147, + "median": 0.21746353579124, + "p90": 0.34538740233748155, + "mean": 0.23503983543721835 + }, + "top25": { + "count": 4800, + "p10": 0.46438300527426274, + "median": 0.5649686672386036, + "p90": 0.6741052273899216, + "mean": 0.5682829621251091 + }, + "top50": { + "count": 4800, + "p10": 0.7257072891491935, + "median": 0.7975855882730909, + "p90": 0.8583120658561214, + "mean": 0.794721678938792 + }, + "bottom25": { + "count": 4800, + "p10": 0.04389082141962781, + "median": 0.06803373425718254, + "p90": 0.10318453074095452, + "mean": 0.07107512763171646 + }, + "heads90": { + "count": 4800, + "p10": 10.0, + "median": 11.0, + "p90": 13.0, + "mean": 11.273125 + } + } + }, + "joint": { + "Q": { + "weak_half": { + "count": 32000, + "p10": 0.0225396528840065, + "median": 0.08576029911637306, + "p90": 0.23477475494146346, + "mean": 0.11232942398573505 + }, + "strong_half": { + "count": 32000, + "p10": 0.05655890852212906, + "median": 0.20693699270486832, + "p90": 0.5206475317478181, + "mean": 0.2704283235834446 + }, + "random_half": { + "count": 32000, + "p10": 0.039973394572734834, + "median": 0.15583113580942154, + "p90": 0.4062000095844269, + "mean": 0.21630935878241145 + }, + "all_heads": { + "count": 32000, + "p10": 0.06481254324316979, + "median": 0.22428051382303238, + "p90": 0.5543476462364201, + "mean": 0.2957394429202832 + } + }, + "K": { + "weak_half": { + "count": 32000, + "p10": 0.028515849635004996, + "median": 0.08704467490315437, + "p90": 0.23199979960918432, + "mean": 0.11572287764720386 + }, + "strong_half": { + "count": 32000, + "p10": 0.07840781956911087, + "median": 0.19813165813684464, + "p90": 0.45838963985443115, + "mean": 0.2555373763643147 + }, + "random_half": { + "count": 32000, + "p10": 0.05652601756155491, + "median": 0.1595868319272995, + "p90": 0.38752546906471264, + "mean": 0.21508053492175533 + }, + "all_heads": { + "count": 32000, + "p10": 0.08563644587993621, + "median": 0.21988031268119812, + "p90": 0.4996467530727399, + "mean": 0.2871456688197213 + } + }, + "V": { + "weak_half": { + "count": 32000, + "p10": 0.02640126012265682, + "median": 0.07635688036680222, + "p90": 0.21732283532619479, + "mean": 0.10623245530284475 + }, + "strong_half": { + "count": 32000, + "p10": 0.07369660735130311, + "median": 0.16931316256523132, + "p90": 0.4663332492113114, + "mean": 0.23960510214965325 + }, + "random_half": { + "count": 32000, + "p10": 0.05152657963335514, + "median": 0.13437538594007492, + "p90": 0.39179082512855534, + "mean": 0.20286414045983112 + }, + "all_heads": { + "count": 32000, + "p10": 0.08243615254759788, + "median": 0.19092711061239243, + "p90": 0.5185699462890625, + "mean": 0.2683242328287452 + } + }, + "QKV": { + "weak_half": { + "count": 32000, + "p10": 0.03147876150906086, + "median": 0.09034762158989906, + "p90": 0.23353940397501005, + "mean": 0.11618917352170684 + }, + "strong_half": { + "count": 32000, + "p10": 0.09105055332183838, + "median": 0.2091781497001648, + "p90": 0.48030592501163516, + "mean": 0.26540057950647317 + }, + "random_half": { + "count": 32000, + "p10": 0.06186078600585461, + "median": 0.16420399397611618, + "p90": 0.39710947275161745, + "mean": 0.2216180787065241 + }, + "all_heads": { + "count": 32000, + "p10": 0.09929505959153176, + "median": 0.22908245772123337, + "p90": 0.5173942029476166, + "mean": 0.29365509447891963 + } + }, + "JS": { + "weak_half": { + "count": 4800, + "p10": 0.0002201629729825072, + "median": 0.001413182239048183, + "p90": 0.0061596495099365866, + "mean": 0.002527013004010179 + }, + "strong_half": { + "count": 4800, + "p10": 0.004459654539823533, + "median": 0.02574532199651003, + "p90": 0.08861614987254145, + "mean": 0.038170266890164915 + }, + "random_half": { + "count": 4800, + "p10": 0.002647196548059583, + "median": 0.013814802747219801, + "p90": 0.05524100922048097, + "mean": 0.02303206111582123 + }, + "all_heads": { + "count": 4800, + "p10": 0.0049355274066329, + "median": 0.028837326914072037, + "p90": 0.10028255358338357, + "mean": 0.04305102449328388 + } + }, + "sqrtJS": { + "weak_half": { + "count": 4800, + "p10": 0.014837889559566974, + "median": 0.037592314183712006, + "p90": 0.07848339229822168, + "mean": 0.042642886323592394 + }, + "strong_half": { + "count": 4800, + "p10": 0.06678063943982125, + "median": 0.16045348346233368, + "p90": 0.2976846575737, + "mean": 0.17296267848374555 + }, + "random_half": { + "count": 4800, + "p10": 0.051450909674167634, + "median": 0.11753639206290245, + "p90": 0.2350340485572816, + "mean": 0.1322700569255782 + }, + "all_heads": { + "count": 4800, + "p10": 0.07025331035256385, + "median": 0.1698155626654625, + "p90": 0.31667419970035554, + "mean": 0.183035157812119 + } + } + }, + "counts": { + "links": 92, + "texts": 4, + "layer_transitions": 23 + }, + "sample_index_key": [ + "fresh-en-1", + "fresh-en-2", + "fresh-zh-1", + "fresh-zh-2" + ], + "winner_identity_file": "winner-identities.csv" +} diff --git a/results/head-dominance/audit-output.txt b/results/head-dominance/audit-output.txt new file mode 100644 index 0000000..1f38d0a --- /dev/null +++ b/results/head-dominance/audit-output.txt @@ -0,0 +1,33 @@ +{ + "passed": true, + "transitions": 23, + "texts": 4, + "link_observations": 92, + "primary_observations_per_metric": { + "Q": 32000, + "K": 32000, + "V": 32000, + "QKV": 32000, + "JS": 4800, + "sqrtJS": 4800 + }, + "undefined_zero_total_per_metric": { + "Q": 0, + "K": 0, + "V": 0, + "QKV": 0, + "JS": 0, + "sqrtJS": 0 + }, + "full_attention_js_links": 24, + "head_sum_controls": 96, + "canonical_forward_relative_error": 0.0, + "all_raw_concentration_and_masks_recomputed": true, + "pooled_concentration_and_joint_summary_recomputed": true, + "original_primary_summaries_unchanged": true, + "sqrt_JS_scale_sensitivity_recomputed": true, + "retained_baseline_cache_files": 100, + "checkpoint_sha_matches_upstream": true, + "prior_official_reference_check_passed_same_implementation": true, + "prior_official_reference_tests": 22 +} diff --git a/results/head-dominance/audit.json b/results/head-dominance/audit.json new file mode 100644 index 0000000..1f38d0a --- /dev/null +++ b/results/head-dominance/audit.json @@ -0,0 +1,33 @@ +{ + "passed": true, + "transitions": 23, + "texts": 4, + "link_observations": 92, + "primary_observations_per_metric": { + "Q": 32000, + "K": 32000, + "V": 32000, + "QKV": 32000, + "JS": 4800, + "sqrtJS": 4800 + }, + "undefined_zero_total_per_metric": { + "Q": 0, + "K": 0, + "V": 0, + "QKV": 0, + "JS": 0, + "sqrtJS": 0 + }, + "full_attention_js_links": 24, + "head_sum_controls": 96, + "canonical_forward_relative_error": 0.0, + "all_raw_concentration_and_masks_recomputed": true, + "pooled_concentration_and_joint_summary_recomputed": true, + "original_primary_summaries_unchanged": true, + "sqrt_JS_scale_sensitivity_recomputed": true, + "retained_baseline_cache_files": 100, + "checkpoint_sha_matches_upstream": true, + "prior_official_reference_check_passed_same_implementation": true, + "prior_official_reference_tests": 22 +} diff --git a/results/head-dominance/checksums.json b/results/head-dominance/checksums.json new file mode 100644 index 0000000..812aae4 --- /dev/null +++ b/results/head-dominance/checksums.json @@ -0,0 +1,127 @@ +{ + "REPORT.md": "7980011dacd7714dbdca3db31452a1c0f44eaf9ef7fc45542de9726bea5f8126", + "analysis-summary.json": "ac0284de9aeb883b680c6e84158141f8d378b88378d5c802c698d8e9635544f0", + "audit-output.txt": "8fc108959206f16c4723bfc6e33f692cc12ad4c67ab0cb6b8938c120f62356de", + "audit.json": "8fc108959206f16c4723bfc6e33f692cc12ad4c67ab0cb6b8938c120f62356de", + "concentration-by-link.csv": "ff214fabf445fe0e7e00bf725268594d654f64bfae2999d948321a7ea128c7db", + "concentration-curve.csv": "223fc728a9910fc3925f842884116f633c3acebd5962955dea5af19867055411", + "external-caches.json": "b2f4a4c8d136ef2341dc320168069c20a4f75f6d2420d65a4f3b8135dd0ac246", + "figures/continuous_dominance.pdf": "00db76cea6528508592f63658f945b778d12683414f1f300e10a24d097cf6a39", + "figures/continuous_dominance.png": "c342eed6dc4d9fa4bc886a893a13c987915b8eb821290b7d420140ab0c1e63f4", + "figures/continuous_dominance.svg": "5d66a9c68bae761213cf2a58b40e9630eeb529b15228c8555c591be9a3896336", + "figures/joint_removal_effects.pdf": "b35132d3e1cf3d202bc6695e64f64c9e495196bc6c9a92659ccf65aceb566040", + "figures/joint_removal_effects.png": "7dc26da9fda80e62c6445c5dee29fe52c06146dea0cd65d444755cf0eb5bb729", + "figures/joint_removal_effects.svg": "03e6b299c1dbfa703ac72cbc7e9749e3173b649a22d829316dd5b5dccbe8728d", + "figures/ranked_influence_distribution.pdf": "cdc72f550d4955a389158de8b717ac8769fe1d6ba64fccbebdbc6722f436a1e8", + "figures/ranked_influence_distribution.png": "040c231492e349f28d64408c3a1c6ccdbca20cbf07d580ac85c527219ddcb5f3", + "figures/ranked_influence_distribution.svg": "fafddde477a6279f26b269fd004d53463ca801eed8b2c30c8c1dac11852a0bcd", + "figures/retrieval_scale_sensitivity.pdf": "56a2f5655196018f4310dbccb29cb55c314111007598491496c9a36256958525", + "figures/retrieval_scale_sensitivity.png": "4f2dac5fdb5506ee5282d459dcb86cc6b58fac05d494d6409c29ea6924365add", + "figures/retrieval_scale_sensitivity.svg": "f51803e68df40df6dda91bce7236c999f405ccf30be56e7819242147a5e9a5b8", + "final-tests.txt": "7209dc75b46bf7814ce691cf85f22323bea97edb5d866e0a507368897e4bf4f2", + "head-sum-controls.json": "fd7a9a79730d213be75096cd90ac01e46e1c38f269f31d5c308349d958e2169e", + "joint-by-link.csv": "e1c132aa2fd5b45191296bc83c4201c545d7d242474a807505ef0ca30066e5d6", + "mainline-activation-tests.txt": "58dce65c11865d5391f0ef46ebc7c33e61b2795e927e8b18226e241cd477807c", + "observations.npz": "9f2a2a911a3ad21d465a299917249d2a970eb32968eade8f0b6d7895ce575c0b", + "primary-analysis-summary.json": "a201ad9938a00471bd9e9e8d28d89bdd4a1e814e7448352598ff800b3dec627f", + "prior-official-reference-check.json": "e82b0d8634ccae0671269587da960e06cce33e0a0e2ed15db15d946f6399d9ed", + "protocol.json": "c8ce23c230aced62bf3609c5c90dd4aca9d20dfd49a5803839aa62e8749110df", + "raw/fresh-en-1-link-00.npz": "5ebc7a6ca028da693a9b9ae0d615ca2783553c28912b8d51150d3e932386c00c", + "raw/fresh-en-1-link-01.npz": "45ad959d3c98b0c19345dbc3c0183477dbaa52b1e0b4c0688177c6f2605e216c", + "raw/fresh-en-1-link-02.npz": "ffb16b2e53428a6cd7dd60c08424f519cae1ce9a1ef0e0de8314d62217cbdda7", + "raw/fresh-en-1-link-03.npz": "e35089f7bc4cfe0d2b94030f1c39d8dfe6d4af292063c2952e88f77b6ab22b11", + "raw/fresh-en-1-link-04.npz": "ead40290d633aeabf5d2365dfcd7d3ade3490cd2a3d2238aae360caf69110438", + "raw/fresh-en-1-link-05.npz": "f02ca1564d210292d0d4698ee2c90dcc9519962b9f17fcaa399b774082e8923d", + "raw/fresh-en-1-link-06.npz": "a936f7fd99905cd1a93a646e267d7204581ff2ddfd88da700261dbcc6af1e6e7", + "raw/fresh-en-1-link-07.npz": "00b240fee2888157bb9f9d6a0e896f6b5e34a22cd690947a91124bb127c4fe17", + "raw/fresh-en-1-link-08.npz": "378f714918265f5109e3e93c72d4930d3b28e0d739138a085c08c701e24ac2a5", + "raw/fresh-en-1-link-09.npz": "351bfbe4e25567be21458f0c4091ce1cd8819c3d4d959b3a199edba06277b28d", + "raw/fresh-en-1-link-10.npz": "33b76293fe9cea3c198408b77981f88c866271ae6d4396c1269de402ff02f124", + "raw/fresh-en-1-link-11.npz": "5ad5eb51f9a3faa6634a6aa3328c663dda5f282462144fadf5c06a305976cad2", + "raw/fresh-en-1-link-12.npz": "4aca02df7c64a815fe65ebf09347c7e4deb47a797fff2aaaaaeb2fe84149fe54", + "raw/fresh-en-1-link-13.npz": "190222021031d7db844c30f0868e693e0999209dfc26826071282411b9cc7234", + "raw/fresh-en-1-link-14.npz": "e2a7fcafc95e6ee1caab2cce5b089a73d9b51aad3fb3ad6dcb898d1932ca874c", + "raw/fresh-en-1-link-15.npz": "83cf132f28ccb7e6da89b83797182a0b7a74bfe52d84510d90763622f4139289", + "raw/fresh-en-1-link-16.npz": "a5459a8f007d9b2669a76532554a9541d77ce9dd71d714b2372ddcc56471380c", + "raw/fresh-en-1-link-17.npz": "b5770f2533eb834e8b51c537c5563b8541152adb58a39d986b842a0cc5a1a556", + "raw/fresh-en-1-link-18.npz": "87577b06589afcbc0c51663ef82d3be2a53730b91834b748e1e08c9e60093153", + "raw/fresh-en-1-link-19.npz": "0b90ce9842b6b590f4a7b1aaed0a7629c07e66ffada6e15313edafd63db3564f", + "raw/fresh-en-1-link-20.npz": "cf248be0f50d3b306a542c009f05b0198d94827141ec76cac500c6e620813dd4", + "raw/fresh-en-1-link-21.npz": "330249a215e9f2f8b3e19d7e18db9e065ca2336982f62a922e3fc8b38ab54526", + "raw/fresh-en-1-link-22.npz": "5decb8e48cc8399c671e39f98418f8920a89259ada3b339bb85d165b6736c747", + "raw/fresh-en-2-link-00.npz": "c66dedbd79b9ea18317c72475b84f21a3207493f25cf42a930a4817f73a8633e", + "raw/fresh-en-2-link-01.npz": "3b307c548be806f629e0c66c12e78d45cb96b28fae97d70a93d7fb056705b740", + "raw/fresh-en-2-link-02.npz": "d21f11c5ab5d554ddbb06d6b937281bc9474f0f45e27ddc83f9c3447b2242e10", + "raw/fresh-en-2-link-03.npz": "b82cba9c387b92c971b193e553d3ba0389c4123556532d397fc6962fc4188a93", + "raw/fresh-en-2-link-04.npz": "cfb5b7b45e7ecb2e1e0eae0f5cc3558b145bd4b1f2784dcc9c17caff5603a1d5", + "raw/fresh-en-2-link-05.npz": "80e5d966082a67bab0dd938567ca57d384e59944c98f05e64a70828b6d3aff97", + "raw/fresh-en-2-link-06.npz": "003fb8c74875cff6def86871280218dd6166b8d248da0f4a2f68beacee2ba2fe", + "raw/fresh-en-2-link-07.npz": "ed6969f4db90d249bc250062a1b717fdb0e3ceef7d4c5fd87e4b223989bde67d", + "raw/fresh-en-2-link-08.npz": "6a3e8ec9f42af7c884dd610d06edb8f815878e7a95d5a4090255fed8098ce57d", + "raw/fresh-en-2-link-09.npz": "6c623a20907e6e176ce3106ddecea592082e2603ecbbc3479266de82cd1f9c49", + "raw/fresh-en-2-link-10.npz": "7d69b262b259562f02517f3c57b978d0cf8f85d93e257f6cb8514847b01924c5", + "raw/fresh-en-2-link-11.npz": "5e334ca69f0944df0e8bfb2308be7d8af30eef553164302eba378e151768aea9", + "raw/fresh-en-2-link-12.npz": "ae51be4496e7d7e7bcf979793c6f60090b0d687a40b845ceb5ff425b0f98c724", + "raw/fresh-en-2-link-13.npz": "88802d963487fda8852900f366443b047769e3a8e3bd347acefc7fcf53709a3e", + "raw/fresh-en-2-link-14.npz": "c2fe9b11d500af37dbf2dd86c2cf0fb4a30dc4e9d45a1f3c2db7ad898f4c51f2", + "raw/fresh-en-2-link-15.npz": "d54cb02cdfba005b7fedd56f5f1c68c1b34cce3b84bcf72c2efb571416dad18e", + "raw/fresh-en-2-link-16.npz": "0dcce2643623dfe6f4e070df34c1050210d3271427776e8d8fab0b1bd99aedeb", + "raw/fresh-en-2-link-17.npz": "f04b29d02b09a2d4bcf67ec00420cf6644680486fac2796f8ec946956150a631", + "raw/fresh-en-2-link-18.npz": "d3b5ce2c4dbadfb8864c698b67fbf8561e01536d03befbd9a80bb3bf9d0341bf", + "raw/fresh-en-2-link-19.npz": "0975878a9b1d0976789ac8123300d47edfe52b0b613cd5c7d29926637e1e89ad", + "raw/fresh-en-2-link-20.npz": "98d1772334e8d20b9412606b04afb31453140122f1bd0849f693ac5f4cbe3125", + "raw/fresh-en-2-link-21.npz": "4202ec9978b272a39578c632898d79a227829146e82eb2a835ef44b56a14959b", + "raw/fresh-en-2-link-22.npz": "bb71e2d13fbff283728037ecff93d87e2475e182fde385216c6d59ab68113f0f", + "raw/fresh-zh-1-link-00.npz": "4f57fd1a43d0c71cf46b9d0fb30e50d08dde04a3e280aee8279385f4b6f1e60e", + "raw/fresh-zh-1-link-01.npz": "dd44a8fb3f22d315db6fda63ca5daaa70fbd2e210cb00633e007e760f99ee20c", + "raw/fresh-zh-1-link-02.npz": "7f3662c3f9badc8d7cd7d7bb312afbc8b7e0e3a3890c64b503040d0d9c11025a", + "raw/fresh-zh-1-link-03.npz": "dcf80f5f5cca2b9799631488e698db21071be280d9521c56f391755dc1669851", + "raw/fresh-zh-1-link-04.npz": "0edfa4ec7a0b77aa0e92815e05678ea5a2080a672ae64b4fbc4cff6b348dd28c", + "raw/fresh-zh-1-link-05.npz": "5927eef6121b9f7818f93ab62a0c9670d18a229784545df268c56dc6b30fa249", + "raw/fresh-zh-1-link-06.npz": "c196a05605c73ad28a1eb18299743e5c2ab9b5edacb31d8bb1afbb5addd90b5a", + "raw/fresh-zh-1-link-07.npz": "53031933bfda41dfe42a230d5cc09e49389b457b55dc6bde3d37fb38818319be", + "raw/fresh-zh-1-link-08.npz": "83ff9806fb45eb0510c2372dbef9747e708291e4f3135dc8251c32686fb167fc", + "raw/fresh-zh-1-link-09.npz": "eaf729d239c95b0ab33e6703bb3251aa6264783cac78b56fc9a19e9f8318b4fe", + "raw/fresh-zh-1-link-10.npz": "5f8e151ce7656350b6e1505356e22fe7fc91a543e19a4315b1be0730e9504c74", + "raw/fresh-zh-1-link-11.npz": "f3b10a05e3857b0f37b0116cdb8a1a71fae2bc9fbc2ccb2ff1f4512b820c3d5a", + "raw/fresh-zh-1-link-12.npz": "0dbd6c93a8213ceab8ea23d9aaec792df0b85cd1e2dea586dc15912ff8ba9b13", + "raw/fresh-zh-1-link-13.npz": "ffd1f4e79cc64dd607ea12068697f75f8216f01520c8b50420b52248f31a1d5e", + "raw/fresh-zh-1-link-14.npz": "bedeb732de606f62e6af6bbc85d90146b0ae0bd0b97889d67e026f7374e89212", + "raw/fresh-zh-1-link-15.npz": "f78fb430d1ccd971b6b4bcb224f97685e99bd2c782246e4479d6ea848d0b1b0d", + "raw/fresh-zh-1-link-16.npz": "18fc345136bbe435d7984298420a05eb1aa15531c40fb7f9c4f4e006b6bda37d", + "raw/fresh-zh-1-link-17.npz": "5f62ac1de5fe094e70e37ffcfd705919ec1cb5513fa200aa98ca609f4ac338b2", + "raw/fresh-zh-1-link-18.npz": "1cffd0b08074027d75997f602e3f42440bc5d2126618da7e74c28a13a34f3bd6", + "raw/fresh-zh-1-link-19.npz": "05d6cfb325e05d1c10f783e997087ce811874bf061d4e402cc9e77b8e6cd3f2f", + "raw/fresh-zh-1-link-20.npz": "be90cb8a9388611e565456cbbd74a606b9cb8ff10e501aa46e0eb052b6c12e81", + "raw/fresh-zh-1-link-21.npz": "227217323faee7c924e969afdf17b0a195ee17520b522705a08d007ce12d75b9", + "raw/fresh-zh-1-link-22.npz": "62326fccae8a66a39ba5cd076464ff4e29d0c0f13492fa588a692d2e8dd8f313", + "raw/fresh-zh-2-link-00.npz": "354e573dad273502055ef4fba2a75ae5f3a44c4eb5e5e20c60ca7c7dd5935f7e", + "raw/fresh-zh-2-link-01.npz": "08670f8293b467842f76608ce3c0ca5cc5a4e23c0b3b1332ad2ebc416be97ec3", + "raw/fresh-zh-2-link-02.npz": "f479c9681dea7c9a20937964eddda0f44fb6c7d896258e8d0be0da897fe4b0f1", + "raw/fresh-zh-2-link-03.npz": "5b59356edb780798d52ea8d8819af5f939c9cb0604083bcf8e476b0c210f740d", + "raw/fresh-zh-2-link-04.npz": "5d22e5d7dddf1c13493d5a2c2c3b5b45d68501d66ff5af8c758050a7ee0124d3", + "raw/fresh-zh-2-link-05.npz": "3df26ca141978b608b7071437d56958dcbad03016fc7fa278db1ea45a540f933", + "raw/fresh-zh-2-link-06.npz": "9bc654f5eb7c8d9ee55afcbc0d2afad1fc2f0472101abadc3c2859d14532c72a", + "raw/fresh-zh-2-link-07.npz": "7b1f9be7a454c0c9a52813c863c95c82a6c0d9f2b703eb2df827c134e6e0c750", + "raw/fresh-zh-2-link-08.npz": "7d0dea896354ed981fbd4da289962d20cac2b89dddc74ab6be2ff80839933bcd", + "raw/fresh-zh-2-link-09.npz": "0d785426c74e4ca6adfd09fdf7999295eeddb8502251494696ef8fa51fab8bc8", + "raw/fresh-zh-2-link-10.npz": "0253a2fe6dc2a429e79bbc42ee8ae36605577df417e84154c6accd2ea674ea37", + "raw/fresh-zh-2-link-11.npz": "0038f167bc8bb0d082992b138c5e7d4c9b828b387340401d09a19e5a67d232f9", + "raw/fresh-zh-2-link-12.npz": "67a18b021e0ec6e7882adc7668354b66c65ba4a967767915f750277470869e46", + "raw/fresh-zh-2-link-13.npz": "a4083495c85f027662c6c788a3084662b0f2b64ad0c2505e521b38dce35a9fb3", + "raw/fresh-zh-2-link-14.npz": "be9496bbe6fc6628e89e712d3196faff89560769fd42956aa147f78f265019b9", + "raw/fresh-zh-2-link-15.npz": "6c50b4c321bf312d2f6f94596923a7921b162e76eae4c2b2663b94fb67e91aa1", + "raw/fresh-zh-2-link-16.npz": "499284cae416c8f117f6cb27e106127c16354600b44a60467308042beafa03f6", + "raw/fresh-zh-2-link-17.npz": "417f34a6a4061a7e6e51b081cb47badb30f729b60c15422ab677a941c3cbd563", + "raw/fresh-zh-2-link-18.npz": "59027a05d715123275ce00e5a999df58cda31807e4b329d91a3f5698c7f2cee6", + "raw/fresh-zh-2-link-19.npz": "58b25fb2c1a0b4bdf6771feafd919b70cfdca2a3ad646c903ab9f6b33c5ebba8", + "raw/fresh-zh-2-link-20.npz": "bcf4820709acfa624acc907e746731968afa7e1b9fc2be9fb64fe56506b14a53", + "raw/fresh-zh-2-link-21.npz": "e5d9f9b182b1151f71773ea263603d7979ba6ce1e374e28c4ab6ad31b9788caa", + "raw/fresh-zh-2-link-22.npz": "d8ad49c825823916d7208672ab86e002e32b0f2b49e2ecbf6238b8244a8487fb", + "report-output.txt": "9b85b125cedd5c6ba94cf747b702f85e6e7512e90fb017754a738a16ffe64b8e", + "run-log.txt": "aa698b663dd9925fe17a57adc203de18b94b1ee559d258d3ff610fa03ebd53c2", + "samples.json": "694a4ca2e59688aed7ce0ab49402c3ef20a99d6f6f6ece8b3bf246423e0c47a7", + "source-hashes.json": "a38b34b0252246622c8386b30792a9d8e069a31352bc6fa23e045826fd2aba64", + "status.json": "ddda7d2d103f93269bdcac559e8a7c76eb0a1c5f5ac5133670297ecc17c99573", + "winner-identities.csv": "882bd3748094344adcbf70fa64569ceaeac189302651d067a452144ece9a1bfa" +} diff --git a/results/head-dominance/concentration-by-link.csv b/results/head-dominance/concentration-by-link.csv new file mode 100644 index 0000000..038d056 --- /dev/null +++ b/results/head-dominance/concentration-by-link.csv @@ -0,0 +1,417 @@ +sample,source,recipient,metric,source_heads,recipient_attention_type,observations,undefined_zero_total,total_count,total_p10,total_median,total_p90,total_mean,max_absolute_count,max_absolute_p10,max_absolute_median,max_absolute_p90,max_absolute_mean,effective_heads_count,effective_heads_p10,effective_heads_median,effective_heads_p90,effective_heads_mean,effective_fraction_count,effective_fraction_p10,effective_fraction_median,effective_fraction_p90,effective_fraction_mean,concentration_count,concentration_p10,concentration_median,concentration_p90,concentration_mean,top1_count,top1_p10,top1_median,top1_p90,top1_mean,top25_count,top25_p10,top25_median,top25_p90,top25_mean,top50_count,top50_p10,top50_median,top50_p90,top50_mean,bottom25_count,bottom25_p10,bottom25_median,bottom25_p90,bottom25_mean,heads90_count,heads90_p10,heads90_median,heads90_p90,heads90_mean +fresh-zh-1,0,1,Q,16,linear_attention,400,0,400,2.429101360589266,3.520149750635028,4.5550282966811215,3.514293308496417,400,0.47488542497158054,0.9837200045585632,1.365362322330475,0.9562569992989302,400,5.687866895642076,7.65898625189508,10.53831450501321,7.863706694223669,400,0.35549168097762973,0.4786866407434425,0.6586446565633256,0.49148166838897933,400,0.03455542133661093,0.0726033575546184,0.12086702416909247,0.0770921629598834,400,0.1842166326588672,0.26346488257079725,0.36494695470739064,0.27145601624143234,400,0.4819759131130914,0.5925477556737547,0.6997333224649719,0.589292752812927,400,0.7258310100300859,0.7970895711531301,0.8535465937350637,0.792762013483378,400,0.04527761954999044,0.06622223626764082,0.09518362673149265,0.06828256727598964,400,10.0,11.0,12.0,11.265 +fresh-zh-1,0,1,K,16,linear_attention,400,0,400,2.6258230035193266,3.464348295237869,4.545622184313833,3.547634291363065,400,0.4927452951669693,0.7190724313259125,1.080737853050232,0.7535221553593874,400,7.399890187205337,9.256685733893061,11.840871677600484,9.448798089478036,400,0.46249313670033354,0.5785428583683163,0.7400544798500303,0.5905498805923772,400,0.023416820252517154,0.048565363958328896,0.07748002529988003,0.050200026455025776,400,0.1576650044791088,0.20577507490508432,0.2828923332979114,0.21337095150308627,400,0.4439259819556893,0.5349321690706872,0.6522081127398792,0.5452988983111056,400,0.7056037079026516,0.7646398746754288,0.8334500440436537,0.7670628738105433,400,0.050077941533532844,0.07441655230876246,0.10566766904618391,0.07582696325555224,400,11.0,12.0,13.0,11.71 +fresh-zh-1,0,1,V,16,linear_attention,400,0,400,1.6618994304095396,2.9560275971889496,5.096769020985813,3.31489211992608,400,0.3370532244443894,0.6078020632266998,1.5504497528076182,0.8222364107891917,400,6.099072061247005,8.689955404862616,11.58607296085583,8.763978263626717,400,0.38119200382793783,0.5431222128039135,0.7241295600534894,0.5477486414766698,400,0.02539790421690647,0.05608042192008958,0.10822372335890323,0.0638103928360236,400,0.15229133635500944,0.21638926612016002,0.3456439351903112,0.23578560584471517,400,0.4592871304770665,0.570193316882557,0.6756661147143489,0.5654061656009641,400,0.7219544738761362,0.7942583639675627,0.8595696834410961,0.7928212505162944,400,0.03671375015579139,0.059905751077522995,0.09501612366996874,0.06277223569909132,400,10.0,11.0,12.0,11.13 +fresh-zh-1,0,1,QKV,16,linear_attention,400,0,400,2.7806776499375703,3.527419180376455,4.623040654230863,3.668958644666127,400,0.5347891211509704,0.8136073350906372,1.184285056591034,0.87097770370543,400,6.906984447085922,8.794509923009494,11.176738874947565,8.869054628913807,400,0.4316865279428701,0.5496568701880934,0.6985461796842228,0.5543159143071129,400,0.028769639339771003,0.0546213130146069,0.08776643205837922,0.05775824421784922,400,0.17313449577164663,0.22849681587593088,0.3151995233777782,0.23505532717527466,400,0.46291476731001363,0.5612733360554396,0.6495100049172878,0.5605365728529281,400,0.7170284639238036,0.7730115849200583,0.8320824929388254,0.7733223586089778,400,0.052385459651589916,0.0737097126227649,0.10188241735582697,0.07529209009772804,400,11.0,12.0,13.0,11.665 +fresh-zh-1,1,2,Q,16,linear_attention,400,0,400,0.4919523543445393,1.0134473060024902,1.5085470150224864,1.0159313781763557,400,0.08705939799547197,0.20679974555969238,0.3549354106187821,0.21840132048353553,400,6.822454189755527,8.598481239074776,10.420966667100178,8.57634675716254,400,0.42640338685972046,0.5374050774421735,0.6513104166937611,0.5360216723226587,400,0.035691098771976376,0.057386994878079314,0.08967984419247192,0.06145501329124079,400,0.15615141462298376,0.20105545653354573,0.2841168347380189,0.2105380875415265,400,0.5005902897160754,0.5803455644643412,0.6681169401685443,0.5836381254101121,400,0.7884049686902198,0.8591029271381095,0.9094417893263294,0.8537247440234702,400,0.016813038207767955,0.02828461758637977,0.054540186859154495,0.03203233886011298,400,8.0,9.5,11.0,9.54 +fresh-zh-1,1,2,K,16,linear_attention,400,0,400,0.722312238404993,1.0673685721121728,1.531943997507915,1.1099483797014544,400,0.1288822755217552,0.20526132732629776,0.3613698065280914,0.22918809283524752,400,7.353096790119529,8.995945617074707,10.826557130257171,8.963614236835804,400,0.45956854938247055,0.5622466010671692,0.6766598206410732,0.5602258898022378,400,0.031856498246925526,0.05190528735344792,0.07839732500959792,0.05586248974389761,400,0.14415505901838727,0.1932984617430117,0.2706608384379229,0.20468892937169264,400,0.4758596355824017,0.5590998802490588,0.6360807056536669,0.5589157781956183,400,0.7783240032912256,0.8414891499411692,0.8890432002987525,0.8374244941741944,400,0.020459486101284795,0.03380453708127341,0.05542032282025878,0.03633820375699517,400,9.0,10.0,11.0,9.945 +fresh-zh-1,1,2,V,16,linear_attention,400,0,400,0.4973134163476061,0.9102987396763638,1.5755363196483816,0.9873435483932735,400,0.09400087669491769,0.1873743161559105,0.3633595824241639,0.20929185537621378,400,6.653126987830839,8.548036858920664,10.615267065905229,8.569869320966436,400,0.4158204367394274,0.5342523036825415,0.6634541916190768,0.5356168325604023,400,0.033817656985912055,0.05811832745038133,0.0936589877279045,0.0622713524657142,400,0.14770778796318934,0.2030391393265193,0.2849489847783121,0.21128165231162654,400,0.4944430848854054,0.5783954342548722,0.6691385367269639,0.5834193552833876,400,0.7867761246770342,0.858135594402403,0.917780688985253,0.8545756163995584,400,0.015093247546088609,0.029487547687592472,0.0536097178897243,0.03234443951371928,400,8.0,10.0,11.0,9.5575 +fresh-zh-1,1,2,QKV,16,linear_attention,400,0,400,0.7504384314408526,1.094116581894923,1.537298795732204,1.1047749286276667,400,0.12243893519043923,0.20843254029750824,0.3249961733818054,0.2172625369206071,400,7.5885696781981755,9.044786656418898,10.899689595781593,9.150839929594042,400,0.47428560488738597,0.5652991660261811,0.6812305997363496,0.5719274955996276,400,0.03119546835705092,0.051265039620679076,0.0738956636306833,0.052284029184004356,400,0.14234151706724504,0.18683652948205282,0.25902377477733723,0.19491383301359225,400,0.4762433814041551,0.5585166073779885,0.6240611452390602,0.5537778694657804,400,0.7829855065671689,0.8451482215064245,0.890195529357141,0.8392779151459865,400,0.0196607747001165,0.033737600841906426,0.056869627611915725,0.03607052435295721,400,9.0,10.0,11.0,9.88 +fresh-zh-1,2,3,Q,16,full_attention,200,0,200,0.8348475917766336,1.1361491297720931,1.6960345567553303,1.2002219437780877,200,0.22832624167203902,0.2860960364341736,0.39249175786972046,0.30112025029957296,200,6.130382096140542,8.164131992339712,9.763619471799924,8.049060555424102,200,0.38314888100878386,0.510258249521232,0.6102262169874952,0.5030662847140064,200,0.04258243760755125,0.06398615784783833,0.10733041697722742,0.06998809959298968,200,0.18845946241609016,0.2530277699392792,0.3380382312714261,0.25834641383503315,200,0.5274206551766942,0.5801220810185626,0.6429009074492729,0.5817870850207154,200,0.7886696609804783,0.8367034920742586,0.8804709973439675,0.8329291960697924,200,0.014185908410063007,0.037601374322041664,0.0648660315307585,0.03854475190086285,200,9.0,10.0,11.0,10.18 +fresh-zh-1,2,3,K,16,full_attention,200,0,200,0.9481974663154689,1.123923289414961,1.459433244937099,1.1702006854029605,200,0.23184000551700593,0.27220672369003296,0.33163814544677733,0.2769536828994751,200,7.221345906722338,8.219899876053379,9.556294933535497,8.338902767051964,200,0.4513341191701461,0.5137437422533362,0.5972684333459686,0.5211814229407478,200,0.044952877192840295,0.06310278749599307,0.08104551970609353,0.06266317998144431,200,0.1913092149796528,0.24399852331036923,0.2838395833820304,0.2402448643049889,200,0.5243109110766915,0.5837120082633487,0.6300260713973377,0.5794902081869165,200,0.8005518168893325,0.8431713945495234,0.8751113885489876,0.8372445372576465,200,0.014816394280292371,0.03623670050698838,0.06232520797438394,0.03764931008410226,200,9.0,10.0,11.0,10.16 +fresh-zh-1,2,3,V,16,full_attention,200,0,200,1.0673203796497546,1.2973932593886275,2.2399458701722317,1.463776254808763,200,0.2684808224439621,0.3130621016025543,0.4607035547494887,0.34213996678590775,200,6.888963030484107,8.57069360970703,9.807240859802178,8.492872798802923,200,0.4305601894052567,0.5356683506066894,0.6129525537376361,0.5308045499251827,200,0.04209715538255347,0.057788472076842456,0.08817622823015829,0.06118871430043193,200,0.1942415822527857,0.2372664246848083,0.29519498694717244,0.239932479346568,200,0.5030105766020796,0.5655827050424824,0.6195335311242982,0.5668923081616112,200,0.7711891554112194,0.8158122185073091,0.8619132870200089,0.8139867859409078,200,0.017300658186363126,0.04459526510423922,0.07570313009017611,0.044977364346506066,200,9.0,11.0,12.0,10.7 +fresh-zh-1,2,3,QKV,16,full_attention,200,0,200,1.0199183085933328,1.1750049997936003,1.9069260513409971,1.3038114167831372,200,0.250296925008297,0.2935818135738373,0.3817494958639145,0.3097901244461536,200,7.196501267820574,8.441507481465084,9.751193704228024,8.44616190715833,200,0.4497813292387859,0.5275942175915678,0.6094496065142515,0.5278851191973957,200,0.04272226697397949,0.059693070665254866,0.08155381570809245,0.06124391513532672,200,0.19220363243778327,0.2452880404617358,0.28456213418232335,0.24238526627742943,200,0.5093232051398124,0.5697030938067207,0.6206605729117751,0.5686563434922721,200,0.7771778324174323,0.8286977027123357,0.8689578331982056,0.823016277118345,200,0.015305978073171437,0.04065693749556967,0.07249220059719527,0.041722741269430416,200,9.0,10.0,12.0,10.465 +fresh-zh-1,2,3,JS,16,full_attention,200,0,200,0.028982629118581828,0.05374104665088453,0.09855992669154144,0.06108389672119415,200,0.010798964649438858,0.024645465426146984,0.08034284934401512,0.03651118309469894,200,1.3175070315523083,3.5480553550417184,6.017363110228346,3.5979467985382336,200,0.08234418947201927,0.2217534596901074,0.3760851943892716,0.2248716749086396,200,0.11061324421376217,0.23398621779410705,0.7429507366244389,0.3257845391446977,200,0.2976556613016461,0.488028777970165,0.8696035028760013,0.5342668192607622,200,0.7064194054950236,0.8101997774989895,0.9495847511402526,0.8153004192781174,200,0.8984980468457894,0.9461568854525526,0.9837933181787005,0.9424611423371019,200,0.0029290946529022268,0.009887020058842282,0.021721080315529443,0.01144397224682556,200,2.0,7.0,9.0,6.06 +fresh-zh-1,2,3,sqrtJS,16,full_attention,200,0,200,0.5068664275109768,0.66347974864766,0.8529259402770549,0.66853969598189,200,0.10391714274883271,0.1569887399673462,0.2834477216005325,0.17618735685944556,200,4.400645222422045,8.823949964387808,10.779013738541964,8.351667359484624,200,0.2750403264013778,0.551496872774238,0.6736883586588728,0.521979209967789,200,0.03229113834557456,0.054216715127082156,0.17572335128539068,0.07490029534993962,200,0.16524507391431523,0.23259758939678682,0.4443199621821767,0.2653715311832368,200,0.4897872200818199,0.5532730641446492,0.6650363202455166,0.5660363140298347,200,0.7454979994578955,0.7984299359235015,0.8522671606190597,0.798027595599657,200,0.04255478964937405,0.06211678658355077,0.08472368152523543,0.06290225602690902,200,10.0,11.0,12.0,11.12 +fresh-zh-1,3,4,Q,8,linear_attention,400,0,400,0.5557760668918491,1.2035991735756397,2.7694889150559905,1.4282717361161485,400,0.10986580029129982,0.2686391770839691,0.9802173316478735,0.385493537671864,400,4.412356865827618,6.5894809376431915,7.294178216213576,6.32901449297088,400,0.5515446082284523,0.8236851172053989,0.911772277026697,0.79112681162136,400,0.013823588797631101,0.03057948390482717,0.1161559317055479,0.0449319615620877,400,0.18930166147926616,0.22759472452826562,0.34535849534858265,0.24612360728472168,400,0.3527067301920025,0.41756426257979695,0.6078362720471945,0.4403330942695859,400,0.6242944902209702,0.6846834001728204,0.8296547046533881,0.6964017941901508,400,0.06480443681296434,0.12528725648034666,0.16017576563531435,0.12294561186246948,400,6.0,7.0,7.0,6.7625 +fresh-zh-1,3,4,K,8,linear_attention,400,0,400,0.7813550390303134,1.1287835463881493,1.8791275978088382,1.2311416709050536,400,0.15441228151321412,0.24663127213716507,0.4807141304016116,0.28828068025410175,400,5.448574678275059,6.86858547196183,7.310594972481833,6.665570492465333,400,0.6810718347843824,0.8585731839952288,0.9138243715602291,0.8331963115581666,400,0.013471742693807258,0.023531870758839682,0.06689628062242073,0.031402245988877704,400,0.18021515251331333,0.21512443309487483,0.30717373045199886,0.228448474221392,400,0.3411882855416954,0.3940215473457226,0.5156435506982462,0.40776819835259387,400,0.6217219186699433,0.6644997460849055,0.7379743245939709,0.6717035536935189,400,0.10195968576323201,0.13333882078099873,0.16010469049043216,0.13233058544649764,400,7.0,7.0,7.0,6.91 +fresh-zh-1,3,4,V,8,linear_attention,400,0,400,0.5568574259057641,1.0929276589304209,2.0630700096487997,1.2380642587295734,400,0.14729319214820863,0.28404104709625244,0.5434242546558382,0.3163391994684935,400,5.2364312861383855,6.215160742536021,7.113703308025048,6.185090559479871,400,0.6545539107672982,0.7768950928170026,0.889212913503131,0.7731363199349839,400,0.01779896561159364,0.04102513374562051,0.0753941873259701,0.044591109741713604,400,0.1944153281993725,0.2544354717187791,0.32521124064773155,0.25746440345878896,400,0.3654906100732668,0.44510639664815854,0.5313035316984108,0.44715327693270523,400,0.640403292413115,0.7079794523062812,0.7647579544221347,0.7042017618810942,400,0.08895730301525785,0.11470029494701212,0.15088125348503126,0.1172794500855358,400,6.0,7.0,7.0,6.71 +fresh-zh-1,3,4,QKV,8,linear_attention,400,0,400,0.7889723796397448,1.2168751917779446,2.1376202166080476,1.377372672157362,400,0.16027085036039354,0.2687840461730957,0.6374220669269562,0.33303765455260875,400,5.075341633710963,6.769595806675813,7.318013294230532,6.520162938019946,400,0.6344177042138703,0.8461994758344766,0.9147516617788165,0.8150203672524933,400,0.013313268019184808,0.025964922332192067,0.08232184636311993,0.03653755501045374,400,0.1833295249601778,0.21763868323456853,0.3135927690484865,0.23266585924959315,400,0.3458837955976494,0.40303658619382643,0.5536054790156133,0.4236755754067282,400,0.6237465215280442,0.6714963247889597,0.7726398501451421,0.682340544061076,400,0.09399344826956293,0.13265386372995686,0.15653309350283023,0.1296820372707155,400,6.0,7.0,7.0,6.8725 +fresh-zh-1,4,5,Q,16,linear_attention,400,0,400,0.5529486530460418,0.9712166800163686,1.3937916854396464,0.9745137749804417,400,0.07718285247683525,0.14298399537801743,0.23485634773969655,0.15007275919429958,400,10.023081933356542,11.68439982919071,12.915023140391108,11.561880889285884,400,0.6264426208347839,0.7302749893244194,0.8071889462744443,0.7226175555803678,400,0.015924487523854707,0.02462315118424866,0.039754359812644816,0.026655923385890602,400,0.1172125475583986,0.14663260613273274,0.1957001125498848,0.15324311732727033,400,0.406328127318147,0.458691855800027,0.5226096313893106,0.46189044288992703,400,0.6924297200750027,0.7354484648183404,0.7767195801918525,0.7350643372448331,400,0.0622921930831271,0.08026613657123874,0.10419169417357137,0.08273736500595881,400,11.0,12.0,13.0,11.9975 +fresh-zh-1,4,5,K,16,linear_attention,400,0,400,0.6115223877946846,0.9048468508990481,1.251787215191871,0.9118778689983447,400,0.07747846469283104,0.12130804732441902,0.19595039635896683,0.1303617076156661,400,10.722570381318834,12.154069436726337,13.307439716828021,12.08792854145712,400,0.6701606488324271,0.759629339795396,0.8317149823017513,0.75549553384107,400,0.013489002802163415,0.021095465388176034,0.03281197361118651,0.022248525525734895,400,0.11181152173275515,0.1383523966997363,0.1774973956714344,0.1421228748304328,400,0.3948537551087122,0.43966621854363186,0.4990546308358602,0.44283328212455025,400,0.6808406730708435,0.7177667740443578,0.7606010570205007,0.718339432237346,400,0.0683096342316758,0.09010447455753517,0.11279447355750542,0.09110416263943287,400,12.0,12.0,13.0,12.21 +fresh-zh-1,4,5,V,16,linear_attention,400,0,400,0.3245677670900477,0.6919851255370304,1.4499126452021303,0.8258946844962338,400,0.056015729904174805,0.10960651189088821,0.25974831581115726,0.1390983904665336,400,9.246466697988279,11.004331334674944,12.412264134831256,10.892907343093466,400,0.5779041686242674,0.687770708417184,0.7757665084269535,0.6808067089433416,400,0.019269886144077084,0.03026490460454602,0.04869271354354161,0.032746298475843,400,0.12717513712770148,0.1628155668133517,0.22181564050232946,0.1691504601454467,400,0.4273240896833351,0.4863091248083924,0.5495400600032926,0.4882786276402555,400,0.7040900313379921,0.7533820660614597,0.8027558507512916,0.7529665700740196,400,0.05255446176086491,0.072717414816377,0.10089424215657365,0.07502067581868756,400,11.0,12.0,13.0,11.735 +fresh-zh-1,4,5,QKV,16,linear_attention,400,0,400,0.6183724389178679,0.8954020673409104,1.3749657900072636,0.9632487819958624,400,0.08114546462893486,0.12070538848638535,0.2128489837050438,0.13513314268551768,400,11.03759630846846,12.259679951088492,13.31912474229267,12.190373056824082,400,0.6898497692792788,0.7662299969430307,0.8324452963932919,0.7618983160515052,400,0.013418676912665448,0.020339411235023842,0.02997274082121978,0.02137383146441303,400,0.11204599139111363,0.13452876494461008,0.16820503191644323,0.13883665744741525,400,0.391955034062306,0.43702979809839326,0.4871559622657572,0.43889545356409654,400,0.6727321025063145,0.717597463372549,0.7565385272636862,0.7156504498371593,400,0.07054508626063702,0.09013293232831338,0.11487727108819931,0.09210415006115148,400,12.0,12.0,13.0,12.245 +fresh-zh-1,5,6,Q,16,linear_attention,400,0,400,0.7430913228541612,1.147736200131476,1.6089440200477838,1.166246029753238,400,0.09927731826901437,0.15613681077957153,0.238798226416111,0.16578936299309135,400,10.863359295960894,12.575643999140802,13.782068203250859,12.405180183911067,400,0.6789599559975559,0.7859777499463001,0.8613792627031787,0.7753237614944417,400,0.010728587354009564,0.018153376635150546,0.03152312244372333,0.020251057164659317,400,0.11037490247005323,0.13572589674515548,0.1794568234279382,0.14208104333730007,400,0.38520976275004104,0.4310202002127746,0.4972803179506637,0.4373981357308564,400,0.6613351021305812,0.7015637143598845,0.7530538230535062,0.7034325937859875,400,0.08583743174901834,0.1097377667327455,0.1366140850223725,0.11104477865767017,400,12.0,13.0,13.0,12.7825 +fresh-zh-1,5,6,K,16,linear_attention,400,0,400,0.8241071481257677,1.122505891136825,1.5527067782357338,1.1691769000247587,400,0.09676236137747765,0.1425568014383316,0.21285407543182375,0.1503893599845469,400,11.75685254912592,13.447297040783358,14.463803209071521,13.271603931688915,400,0.73480328432037,0.8404560650489599,0.9039877005669701,0.8294752457305572,400,0.007080661035434766,0.012655346477137478,0.02406057719815614,0.014290592859103137,400,0.10101409270525082,0.12341164568454366,0.16539557717265407,0.12856649489850935,400,0.35510439307558944,0.39929412378299156,0.4575560002153773,0.40396296680481564,400,0.6271906707564133,0.6686115701156119,0.716526233909435,0.6705015811752727,400,0.1013596870676708,0.13100641981410865,0.15789417174238096,0.13025604262662221,400,13.0,13.0,14.0,13.185 +fresh-zh-1,5,6,V,16,linear_attention,400,0,400,0.6337890403345228,0.9958858909085393,1.638427373580635,1.1012471098068637,400,0.09457711055874825,0.15001538395881653,0.2569383829832077,0.16589635238051414,400,9.968687883963238,12.16123583144756,13.63070961812831,11.963441490701625,400,0.6230429927477024,0.7600772394654725,0.8519193511330194,0.7477150931688515,400,0.011588002134642065,0.021043841449477144,0.04033508188148871,0.023928837109109678,400,0.1149699135082047,0.1444550466441079,0.2017398771423816,0.15336156900852616,400,0.38979639155693535,0.44726902793020384,0.5203732030368934,0.45182293508115234,400,0.6632123907349297,0.7105725281533102,0.773868696376879,0.7145026094714317,400,0.07429125170008277,0.105520159984926,0.1361598437108261,0.10503434568888251,400,12.0,13.0,13.0,12.5575 +fresh-zh-1,5,6,QKV,16,linear_attention,400,0,400,0.914624169562012,1.1974874115549028,1.5274685110896826,1.2126480673323385,400,0.10662956014275551,0.14808592945337296,0.20532605648040772,0.15299008158966898,400,12.170506927001252,13.481737822544865,14.33758741526606,13.349694981320868,400,0.7606566829375783,0.8426086139090541,0.8960992134541288,0.8343559363325542,400,0.007729857418288237,0.012452716928912928,0.020977026812987052,0.01360336749422371,400,0.10306485918952771,0.12136694474347459,0.15441393193338307,0.12612433272925463,400,0.36371203636915284,0.39822686076617186,0.44813317509168193,0.402734832383121,400,0.6339899147427374,0.6678814265368334,0.709855463681161,0.6699628866826453,400,0.10638763102622426,0.1315459353030145,0.15282281011257037,0.13035173953935147,400,13.0,13.0,14.0,13.185 +fresh-zh-1,6,7,Q,16,full_attention,200,0,200,0.6967215406475589,0.8647677625995129,1.1318519309628754,0.8975616289848404,200,0.17643480896949768,0.21954673528671265,0.2563210129737854,0.21804183520376683,200,6.618313132011688,8.77779598215449,10.418830134116405,8.637575406800561,200,0.4136445707507305,0.5486122488846557,0.6511768833822753,0.5398484629250351,200,0.03571207334000921,0.05485249845415002,0.09450269802868984,0.060977296438866214,200,0.19208097882619532,0.23602997737094536,0.32965308733567916,0.2506137886475929,200,0.49666814431412143,0.5572856142656408,0.6170435045908121,0.5553046780869008,200,0.7480730385048123,0.7980640258486347,0.8526203239706098,0.7981134888980815,200,0.039491579187455876,0.06310500190242457,0.08514610754877759,0.06341188559252524,200,10.0,11.0,12.0,11.125 +fresh-zh-1,6,7,K,16,full_attention,200,0,200,0.7858375686686487,0.924020110629499,1.1293065979145462,0.954063942823559,200,0.17995065450668335,0.21916107088327408,0.2575859248638153,0.2186301603913307,200,7.88689255300789,9.09079278609248,10.156849093731527,9.027217101795987,200,0.4929307845629931,0.56817454913078,0.6348030683582204,0.5642010688622492,200,0.038354240686290286,0.05066854281792081,0.06857903905460921,0.052576098667474855,200,0.19012447386201045,0.2325570240108474,0.2743220075911339,0.23186636101326005,200,0.5061027063145299,0.5477502919164697,0.5906733363766902,0.548262812827514,200,0.7415877916068663,0.7925856942838999,0.8355623230446305,0.7916615319762194,200,0.04248253715574244,0.06732571147416139,0.0919246176836225,0.06689000815217126,200,10.0,11.5,12.0,11.3 +fresh-zh-1,6,7,V,16,full_attention,200,0,200,0.9605062095448376,1.1428413768298924,1.3685777499340468,1.1591908320144284,200,0.18395001739263536,0.23824387788772583,0.2855415284633636,0.23651585191488267,200,7.849490984354433,9.720638326227162,11.491538283414172,9.75516520332515,200,0.49059318652215206,0.6075398953891976,0.7182211427133858,0.6096978252078219,200,0.0261552944254837,0.04306600870793091,0.06922333334667033,0.04511758049993119,200,0.15221090312945088,0.196303315084899,0.2714761761644639,0.2081661712960845,200,0.46401879303112714,0.524439052499995,0.6033623833479212,0.5289478617304235,200,0.722501849673635,0.7781866770312609,0.829828023059304,0.7765126503548125,200,0.04951178031102099,0.07199580649954183,0.09990532346205719,0.07328482140146445,200,10.900000000000002,12.0,12.099999999999994,11.62 +fresh-zh-1,6,7,QKV,16,full_attention,200,0,200,0.8571053378749639,1.0059685087762773,1.183649718435481,1.0213233802010655,200,0.1884602651000023,0.22348716855049133,0.2585281401872635,0.22458526909351348,200,7.7944570256649115,9.303971982080625,10.748833530405351,9.3710458372141,200,0.48715356410405697,0.5814982488800391,0.6718020956503344,0.5856903648258812,200,0.03256903217548453,0.047979708973495835,0.07018273007133141,0.048917588117041826,200,0.1801807823689268,0.22031704948065356,0.2803393183923416,0.22345551685200246,200,0.48813704243826284,0.5326324197050982,0.5923833864598406,0.5361561869350651,200,0.7333737956269141,0.7852361151915166,0.8346699733112655,0.7834009035789988,200,0.046062572291322504,0.07073412362494022,0.09617805857215993,0.07015698184671398,200,10.0,12.0,12.0,11.47 +fresh-zh-1,6,7,JS,16,full_attention,200,0,200,0.0218641472303716,0.04420311772855712,0.07254370295777335,0.046871714137829486,200,0.007316942419856787,0.015788674354553223,0.05015565417706966,0.022694225640734658,200,1.6588479180249318,4.120520295512728,6.2380292228952285,4.145432773729307,200,0.10367799487655824,0.2575325184695455,0.3898768264309518,0.2590895483580817,200,0.10432844124517657,0.19221927366868563,0.5763515831250512,0.2563242193465016,200,0.2611877284354106,0.3885966051481249,0.77089108764741,0.45008556331872107,200,0.7068351247273077,0.8194089704456659,0.9291147357965708,0.8191980249013184,200,0.9010202346431437,0.953330766846342,0.981680517137843,0.9466400398116648,200,0.003264056999811985,0.009318286972849775,0.022254085990137965,0.011427831522990244,200,3.0,6.0,8.0,6.045 +fresh-zh-1,6,7,sqrtJS,16,full_attention,200,0,200,0.43271681810729207,0.6038041174178943,0.7724204423837363,0.6074852672725684,200,0.08553836718201638,0.12564750388264656,0.22395427972078322,0.14068200470879674,200,5.5313715726687915,9.05250186330765,11.02686062187503,8.642337837729093,200,0.34571072329179947,0.5657813664567282,0.6891787888671894,0.5401461148580683,200,0.030066917695381033,0.05116493326388823,0.12617347149173305,0.06521773414851664,200,0.15726624767737105,0.20925916539896955,0.3767178850854693,0.23498832827314786,200,0.4891220651431828,0.5739843609213888,0.6664845470243277,0.5781163231024855,200,0.7490316363110263,0.809991048466753,0.8609176690120646,0.8073749254513416,200,0.04298112131965184,0.06218602194312592,0.08717511464006347,0.06344463270935115,200,10.0,11.0,12.0,10.985 +fresh-zh-1,7,8,Q,8,linear_attention,400,0,400,0.9590097105130553,1.536447413265705,2.8689802177250385,1.7718030927423387,400,0.2201713100075722,0.350384846329689,0.7897801935672762,0.4108072089031339,400,5.887995191027056,6.556289210172601,7.03738792472958,6.483028773113856,400,0.735999398878382,0.8195361512715751,0.8796734905911975,0.810378596639232,400,0.01954077549538053,0.0314575077657065,0.05124239507573904,0.03432354139635862,400,0.19217184731809483,0.22380172267317855,0.285121693929006,0.2321920082524678,400,0.36168901676011517,0.4055534217669472,0.46935854299472085,0.4116085790974441,400,0.6456903072351644,0.6991136955869005,0.7464143915027599,0.6965638856307459,400,0.09023509116070617,0.11262579913286752,0.13680010371064633,0.11273767662286559,400,6.0,7.0,7.0,6.75 +fresh-zh-1,7,8,K,8,linear_attention,400,0,400,0.2979481376707554,1.323602693155408,2.134860857576132,1.3736332048056648,400,0.056699202582240114,0.2807885557413101,0.4808495283126831,0.29184197531547396,400,6.17633895332198,6.747250638628673,7.182301767760164,6.716318555637633,400,0.7720423691652475,0.8434063298285841,0.8977877209700205,0.8395398194547041,400,0.01626416207323056,0.026524062350611253,0.0421808272578277,0.027959279250897032,400,0.18107303321688972,0.20608203155997337,0.2459019637759514,0.21177290855723657,400,0.3479226112740468,0.3884903159298687,0.445413989857238,0.3921656222076625,400,0.6327225785671413,0.6830521555684579,0.7329194726699658,0.6841702161399162,400,0.09330418100990448,0.11866693654289341,0.14254238060485677,0.11922903165646248,400,6.0,7.0,7.0,6.8625 +fresh-zh-1,7,8,V,8,linear_attention,400,0,400,0.5749102413654328,1.3443213794380426,2.2224256038665784,1.3652205040352419,400,0.13838060349225997,0.33455508947372437,0.667331808805466,0.37192271476611494,400,4.618268312269324,6.148824305883943,7.047258111244446,6.002404129055444,400,0.5772835390336655,0.7686030382354929,0.8809072639055557,0.7503005161319305,400,0.019313330009964408,0.04300889178036811,0.10460852140022825,0.05208423583689962,400,0.19349817708912645,0.2477619106951734,0.37722684758310093,0.2655965360587777,400,0.36569162040163705,0.43765681207277635,0.5450941761665091,0.44696303511275415,400,0.6460117608640545,0.7203376222935364,0.8038236040088341,0.7252946084484383,400,0.06500499349972674,0.0977581707612199,0.13500608891329802,0.09991695178335462,400,6.0,6.0,7.0,6.4675 +fresh-zh-1,7,8,QKV,8,linear_attention,400,0,400,0.9274983126670122,1.5033815149217844,2.265283572673798,1.6018475520424544,400,0.20073575228452684,0.34915004670619965,0.5573251724243166,0.36830392001196743,400,5.881230558451355,6.533175768060082,7.0334002379498095,6.50228709856116,400,0.7351538198064194,0.8166469710075103,0.8791750297437262,0.812785887320145,400,0.019632933616512954,0.03207422231802123,0.051465729615420445,0.03381865634780095,400,0.19191701177455783,0.22466129502394616,0.27854774459632486,0.23210277523306297,400,0.36241479805036053,0.4072979515115139,0.4647608255404677,0.4102082544905171,400,0.6531008908578214,0.6955317920204706,0.7449502556564871,0.6982900089685914,400,0.09255299111874246,0.11526921139042773,0.13980834372907236,0.11550857560302565,400,6.0,7.0,7.0,6.805 +fresh-zh-1,8,9,Q,16,linear_attention,400,0,400,0.6906953494995832,0.9759602006524801,1.3041144616901874,0.9891591147868894,400,0.08369906172156334,0.13099230080842972,0.21312458366155626,0.14000689126551152,400,10.640070932820711,12.624016402558667,13.899533292265506,12.399101890486236,400,0.6650044333012944,0.7890010251599167,0.8687208307665941,0.7749438681553897,400,0.010074521166483894,0.01782839516406866,0.03358329824229189,0.02042438161207624,400,0.10595865367818512,0.13220320037299446,0.18576767336076583,0.1407708642559,400,0.3777364081514151,0.4301187530099717,0.5008169400509738,0.43413663616514425,400,0.6544821006745523,0.7046938314194792,0.7681356332649253,0.7078883740361931,400,0.0767036490911563,0.10572786694471734,0.13498928833778048,0.10654524733371065,400,12.0,13.0,13.0,12.6075 +fresh-zh-1,8,9,K,16,linear_attention,400,0,400,0.6804344878531993,0.9005166748538613,1.2611991601064803,0.9440281149331713,400,0.08516289219260216,0.11770259588956833,0.1628935605287552,0.12126477730460465,400,11.553123987583536,12.927051575026713,14.10695059168498,12.864031807811653,400,0.722070249223971,0.8079407234391696,0.8816844119803112,0.8040019879882283,400,0.008946183752776801,0.015847639061143302,0.025660532662294917,0.016748542470120687,400,0.10528301606715158,0.1274286612137167,0.15860624460553727,0.1300115002223839,400,0.3701796005108164,0.41601338834864776,0.47160429878421145,0.41986111729636294,400,0.6463018979578277,0.6953485484459442,0.7417257945524633,0.6950382296934545,400,0.08809085513305247,0.1139740629371002,0.14279577639344965,0.1153185605422464,400,12.0,13.0,14.0,12.88 +fresh-zh-1,8,9,V,16,linear_attention,400,0,400,0.4647453504148871,0.8514449596405029,1.3727117770351471,0.8895953201281372,400,0.06001258864998817,0.11238456517457962,0.19505773931741718,0.12199042475782335,400,11.024187380011924,12.555282564476812,13.99311472157551,12.541131170865151,400,0.6890117112507452,0.7847051602798008,0.8745696700984694,0.783820698179072,400,0.009561299214860333,0.01829093275857004,0.030090276553372184,0.019236171223172657,400,0.10527026377123808,0.13150052741416599,0.17795679687692958,0.1384485005812238,400,0.3726471560437381,0.4287579672288325,0.480353291316335,0.42760517432545087,400,0.6503380799099947,0.7024063546843993,0.7511587142359358,0.7022635946203094,400,0.08064818058038067,0.10735369303982276,0.13772310148465183,0.10868890636633426,400,12.0,13.0,13.0,12.66 +fresh-zh-1,8,9,QKV,16,linear_attention,400,0,400,0.7347855348140001,0.9900411260314286,1.2494902014732363,0.9919958586548455,400,0.08569022938609124,0.11771636456251144,0.16533384323120118,0.12244859239086509,400,11.713702290469794,13.366565529301592,14.388408040683208,13.194810407005507,400,0.7321063931543621,0.8354103455813495,0.8992755025427005,0.8246756504378442,400,0.007467084877844461,0.013134438286334798,0.02439484120737057,0.014676947362255674,400,0.09843854179026623,0.11848844116881953,0.16178088108267588,0.12455204762435379,400,0.35718058665988645,0.40166317927502393,0.45818110260033357,0.40485099512404354,400,0.6373949780874816,0.6793543264952187,0.7295639099577741,0.6813777219056544,400,0.09487160548480769,0.11917564610599386,0.1468835260329232,0.12028897695944402,400,12.0,13.0,14.0,12.98 +fresh-zh-1,9,10,Q,16,linear_attention,400,0,400,0.6123335564509035,1.0435317065566778,1.5125911281444133,1.0584334547061007,400,0.09132177904248238,0.1700449287891388,0.31987524628639225,0.1913822599966079,400,8.330971692520261,11.649403297496246,13.425412334713464,11.337067313609955,400,0.5206857307825163,0.7280877060935154,0.8390882709195915,0.7085667071006222,400,0.012784650291930719,0.02489745779476693,0.06136966958039111,0.031073991890478138,400,0.11821138091871673,0.16235856867909693,0.26472613347971885,0.1769987150104201,400,0.39304157101247156,0.46032250596151914,0.5456046701110793,0.46556338315380896,400,0.6634417693869368,0.7173283982910208,0.7769846698882061,0.7202570102649994,400,0.07568759913736456,0.10535935837863662,0.13380928270248182,0.1053257624245752,400,12.0,13.0,13.0,12.58 +fresh-zh-1,9,10,K,16,linear_attention,400,0,400,0.6221934565342963,0.88918552454561,1.1792654864490035,0.9043257557519246,400,0.08381196409463883,0.13706430047750473,0.21694077700376513,0.1454845827911049,400,9.827882173616375,12.3820462481081,13.798569310550421,12.081156863033502,400,0.6142426358510235,0.7738778905067563,0.8624105819094013,0.7550723039395939,400,0.010636036140978848,0.019479568472037107,0.041868076486150714,0.023512858214741295,400,0.11095983883008902,0.15044731324705274,0.22994426094434903,0.1614972259683926,400,0.38147288457538203,0.433554646378014,0.5006927841623557,0.43708378375953133,400,0.6559676147350192,0.6970414070602557,0.7370198129374841,0.6972230948101901,400,0.09488088237493372,0.11299173709311297,0.13875211719644506,0.11502455500710494,400,12.0,13.0,13.0,12.875 +fresh-zh-1,9,10,V,16,linear_attention,400,0,400,0.47369214249774816,0.7299782517366111,1.2106899134814741,0.7964116735389689,400,0.06351463720202447,0.11250527203083038,0.24161839336156846,0.13542296192608774,400,9.180543887958322,12.360010938789257,13.952589667740078,11.894512916256883,400,0.5737839929973951,0.7725006836743286,0.8720368542337549,0.7434070572660552,400,0.00978274803270085,0.019633149177695403,0.049521104768380175,0.026415459372986203,400,0.11021547564934674,0.1470128731388507,0.25165570236062357,0.16637191909497456,400,0.3716573275869789,0.4390875760368995,0.5159379185395504,0.4428726164650124,400,0.6495829617174365,0.7000604343000693,0.7579582032394955,0.7015903073592102,400,0.0808565812463129,0.11397830192274325,0.1406547874755669,0.11254211937468542,400,12.0,13.0,13.0,12.77 +fresh-zh-1,9,10,QKV,16,linear_attention,400,0,400,0.654531497694552,0.9747728491201997,1.2900495495647193,0.9764982675691135,400,0.08734235987067222,0.1456669345498085,0.23176026046276094,0.15643592607229947,400,10.189556966907771,12.572922538627273,14.08108662565195,12.252288871768865,400,0.6368473104317357,0.7858076586642045,0.8800679141032469,0.7657680544855541,400,0.009085069797216454,0.01817173646254427,0.038015776728644736,0.022114843667238367,400,0.10879222365899918,0.14939314066001885,0.22505287597261073,0.15931209065634153,400,0.3715193365667435,0.4242499322184902,0.48985082809961855,0.43016650297444875,400,0.6431422860044751,0.6925475179731045,0.7327877414436106,0.6904168124438282,400,0.09855978110734318,0.1170700936145091,0.14368655431423133,0.1198108417892592,400,12.0,13.0,14.0,12.9925 +fresh-zh-1,10,11,Q,16,full_attention,200,0,200,0.6438421585597098,0.872416652739048,1.0983223179355264,0.872299394311849,200,0.09125480353832245,0.11970581114292145,0.14658514261245728,0.11973702851682902,200,11.609295715854033,12.717099346353653,13.576108291747113,12.622034448184133,200,0.7255809822408771,0.7948187091471033,0.8485067682341946,0.7888771530115083,200,0.011902770702651415,0.017209941516929887,0.02521372964154654,0.01817272391192884,200,0.11934270921725082,0.1386696549751888,0.1576088519494992,0.13837439252480127,200,0.39571255979610603,0.4290339099297917,0.4744535060326238,0.4344109952622634,200,0.6554351464616354,0.6947435535418892,0.7408203238498978,0.6974609658906215,200,0.08965506465009455,0.11771802752401339,0.1443072155958412,0.11752583125971311,200,12.0,13.0,14.0,12.925 +fresh-zh-1,10,11,K,16,full_attention,200,0,200,0.7055036302655936,0.8617449305020273,0.9929532202892004,0.8748279941594228,200,0.12480776831507683,0.18543966859579086,0.2604644924402236,0.19279752627015115,200,7.4114578468262104,9.771738678821816,12.293996588675094,9.741479862135373,200,0.46321611542663815,0.6107336674263635,0.7683747867921934,0.6088424913834608,200,0.020097285230125456,0.04249346901007654,0.07725467161321313,0.047791621176838675,200,0.1477751327880865,0.222982705354772,0.29433197182296517,0.2196017917041651,200,0.44604160298719675,0.5294650952945521,0.6140385059195879,0.5265543030929237,200,0.7029335352420754,0.764345885041203,0.80418678509106,0.7548839829935571,200,0.07141042908830074,0.08865212568336756,0.12031691664018135,0.09273965767175764,200,11.0,12.0,13.0,12.26 +fresh-zh-1,10,11,V,16,full_attention,200,0,200,0.7727446023374797,0.9757196996361017,1.1745961803942917,0.986132373129949,200,0.09506728425621988,0.11816420406103134,0.17792212665081017,0.12691691622138024,200,11.990124619066028,13.203104874705964,14.147263392230133,13.079529608672026,200,0.7493827886916268,0.8251940546691228,0.8842039620143833,0.8174706005420016,200,0.008730832230477678,0.014122440936986825,0.022296483904800397,0.015229241195584086,200,0.10522607325408717,0.12278047010710506,0.15574624628769235,0.12800411431446537,200,0.3752450581506017,0.4060970764309562,0.4595370936555391,0.41101067402953734,200,0.6461414430484381,0.6848193998375264,0.7251867851894941,0.6867201370041193,200,0.10300035509751843,0.12251839269388934,0.14402710472990213,0.12249546665575832,200,13.0,13.0,14.0,13.04 +fresh-zh-1,10,11,QKV,16,full_attention,200,0,200,0.7646997133269906,0.9415694242343307,1.1005946589633822,0.9381568700482603,200,0.10172526240348816,0.14520113170146942,0.17797999531030656,0.14074353359639644,200,10.556494727505484,12.132287690814861,13.526319939468888,12.102763513736924,200,0.6597809204690928,0.7582679806759288,0.8453949962168055,0.7564227196085578,200,0.012191935409086778,0.021252998106209633,0.034377010706376,0.022337764697260877,200,0.11199920705088734,0.14764846719062247,0.19376761916533158,0.1512855342947202,200,0.3976379062115118,0.4519844618754032,0.5082503153261523,0.4519074543448034,200,0.6643751919368126,0.7095712428180596,0.7495152168943192,0.7102956417116113,200,0.08967173405138439,0.11288960000503676,0.13725722023218334,0.1131325445636444,200,12.0,13.0,13.0,12.845 +fresh-zh-1,10,11,JS,16,full_attention,200,0,200,0.016045292872877325,0.028061560849891976,0.0557658998943225,0.03360439002613021,200,0.004152655368670821,0.008428643923252821,0.022947734035551545,0.013680369750363752,200,2.6450400622520918,5.813073237536349,8.279876067241618,5.747200074222649,200,0.16531500389075574,0.3633170773460218,0.5174922542026011,0.35920000463891555,200,0.06216054871369186,0.11682778177670528,0.33709448395412145,0.16609993910311552,200,0.20591628311538554,0.3152610044344926,0.5464516179835875,0.352571257796797,200,0.6040428255496345,0.7240077921739443,0.8732768242837106,0.7241230388127226,200,0.8327742613094211,0.8957278587660789,0.9573599228471149,0.8942875428845476,200,0.011515956437173307,0.030337179298580402,0.0555964302539676,0.032318215880978524,200,5.900000000000002,9.0,11.0,8.335 +fresh-zh-1,10,11,sqrtJS,16,full_attention,200,0,200,0.40895284092985096,0.5534483557567,0.7309623930603266,0.5665644111053553,200,0.0644407406449318,0.09180759266018867,0.15148354917764661,0.10463369278237224,200,7.860647455468036,11.217304580067477,12.886992169252563,10.796342939084912,200,0.49129046596675224,0.7010815362542173,0.8054370105782852,0.674771433692807,200,0.016104150759088904,0.02842451321013062,0.06903769231945556,0.03846723173886477,200,0.1267601433826018,0.16769519761061916,0.27860473981048695,0.18530787381293057,200,0.4231478142339164,0.4907384095410912,0.5951599695983893,0.49822297329287757,200,0.6852193650415406,0.7369754038291347,0.8022838070372944,0.7395950195094779,200,0.07282213999186168,0.1024971813288944,0.13109117593541883,0.10247885237114591,200,11.0,13.0,13.0,12.45 +fresh-zh-1,11,12,Q,8,linear_attention,400,0,400,0.7112594723701477,1.0567578841000795,1.6549084380269052,1.1336770423408598,400,0.18845413625240326,0.30616503953933716,0.47522892951965334,0.3175508876144886,400,5.477104963346159,6.167510382221616,6.804095606981063,6.136919410117219,400,0.6846381204182699,0.770938797777702,0.8505119508726329,0.7671149262646524,400,0.025108933183938236,0.042445689895851846,0.06580367072889871,0.04475103603953981,400,0.22787424220345726,0.2817419783270949,0.3324606188054017,0.2817903592883699,400,0.39344020228911825,0.438959531659748,0.4919223903308211,0.44102890821117285,400,0.6433718848722331,0.685298774397114,0.7227518653916153,0.6853736661239761,400,0.10090703268802823,0.1250137334582667,0.15025944842985134,0.12518718360010553,400,7.0,7.0,7.0,6.9075 +fresh-zh-1,11,12,K,8,linear_attention,400,0,400,0.4825267916545272,0.7918677963316441,1.3957195408642293,0.8667944692494348,400,0.115846898406744,0.20595640689134598,0.35331148505210896,0.2251581295207143,400,5.8434520497440365,6.457064459438968,6.942915252569421,6.42405429819099,400,0.7304315062180046,0.807133057429871,0.8678644065711776,0.8030067872738738,400,0.02175053586340759,0.034136159417198164,0.05272197723250966,0.035819034174482886,400,0.21790518103264905,0.2592537366780431,0.3068653932707722,0.26112305044886935,400,0.3829190329466067,0.41960124232959434,0.46804767912562667,0.4226426128686665,400,0.6373128610452358,0.6706433711325317,0.7101707379814962,0.6725711973067371,400,0.1047777385601039,0.13292024787245432,0.15963932834829628,0.1325006599701059,400,7.0,7.0,7.0,6.9225 +fresh-zh-1,11,12,V,8,linear_attention,400,0,400,0.38287746962159874,0.6756213149055839,1.3400470308959487,0.7780662520381156,400,0.096944247931242,0.17358769476413727,0.38170437216758735,0.21239002059213818,400,5.347247468994016,6.3113622069751845,7.030087510636093,6.269994428450459,400,0.668405933624252,0.7889202758718981,0.8787609388295117,0.7837493035563073,400,0.019709437489822405,0.03822219781231748,0.07087129632330745,0.04145742046065472,400,0.20707489119098008,0.2651902463629251,0.33642074080443696,0.2681936516611965,400,0.37404562197723296,0.4303926733600409,0.49829614935892047,0.4317686004872223,400,0.6381526274537254,0.6841376109522127,0.7322858508963195,0.6842622430361339,400,0.09566515178460988,0.12749238832849555,0.15951941259016467,0.126557255479741,400,6.0,7.0,7.0,6.87 +fresh-zh-1,11,12,QKV,8,linear_attention,400,0,400,0.627941570058465,0.924285352230072,1.3742050785571336,0.9785891493735835,400,0.16097690761089326,0.2543494999408722,0.3713564544916153,0.263376995883882,400,5.842390607133181,6.363882746730194,6.875659918892965,6.35374740963113,400,0.7302988258916476,0.7954853433412743,0.8594574898616206,0.7942184262038913,400,0.02336066898345175,0.03672774202553174,0.052757711452266154,0.037742441588207824,400,0.22775292520472104,0.2693795954833915,0.31176184783773936,0.2694071747400757,400,0.38766180353829344,0.4247318060379533,0.4674584450265044,0.42593835708836436,400,0.6365551962595403,0.6739642650105429,0.7068103225019726,0.6728856396058228,400,0.10982024019556197,0.13175498010059722,0.15648308683555595,0.13196426261836605,400,7.0,7.0,7.0,6.96 +fresh-zh-1,12,13,Q,16,linear_attention,400,0,400,0.4287725748727098,0.6291518090292811,0.9183653243817392,0.6602286203455878,400,0.05749732218682766,0.09600819647312164,0.22757867276668553,0.12448112932499499,400,7.43813104642693,12.054362349920932,13.725438264243468,11.388680405982191,400,0.46488319040168313,0.7533976468700583,0.8578398915152168,0.7117925253738869,400,0.011047915290649094,0.021821357200642225,0.07674093058932209,0.032833729328761045,400,0.10895990306851573,0.15395135420424036,0.3031144707562746,0.1778963871899312,400,0.3805769265171659,0.44918117219250897,0.5767905651099959,0.4604233742475648,400,0.6593403165378923,0.7087942087917141,0.7841835646721264,0.715777538836388,400,0.07192378850252647,0.10222048066287616,0.1311494953473219,0.10159369808777655,400,12.0,13.0,13.0,12.4725 +fresh-zh-1,12,13,K,16,linear_attention,400,0,400,0.3796079422580078,0.5419529881328344,0.7618788884021342,0.5660406191175571,400,0.04488024935126305,0.07666042819619179,0.131016717851162,0.08314666661433875,400,10.67482621767658,12.599542119525275,14.11989090209039,12.47575948850145,400,0.6671766386047863,0.7874713824703297,0.8824931813806494,0.7797349680313406,400,0.008876885840609866,0.017992535572854598,0.033257087895001446,0.020074796585010615,400,0.1032308308909768,0.13959976769114857,0.1941482275072908,0.14532372406218272,400,0.36320403311127286,0.4224121800926574,0.4895725791001946,0.4271854374902489,400,0.635751315318764,0.6901742968222736,0.7499592678652381,0.6925325786849674,400,0.08499378386993076,0.11317662359439792,0.14312498871384305,0.11334308099216585,400,12.0,13.0,13.0,12.7525 +fresh-zh-1,12,13,V,16,linear_attention,400,0,400,0.33805561484768987,0.4774762666784227,0.692503923829645,0.49688007039774673,400,0.04425990618765354,0.07029108703136444,0.12124188691377644,0.07836978056933731,400,9.334433954331507,12.151683782608224,13.6965244645547,11.846482187869459,400,0.5834021221457192,0.759480236413014,0.8560327790346688,0.7404051367418412,400,0.01121197126399782,0.021112663974228417,0.04760557241782967,0.02550273656352748,400,0.1105578998537858,0.14504039721186468,0.22157384391459353,0.1571277179684568,400,0.3847757296627163,0.442563819587272,0.5456142644678524,0.4534863578946043,400,0.6596884432581905,0.7109339960225283,0.7803934258555302,0.715809932558836,400,0.06898549039660547,0.10243430414267252,0.13183404494659434,0.10127319153323804,400,11.0,13.0,13.0,12.4775 +fresh-zh-1,12,13,QKV,16,linear_attention,400,0,400,0.46751984632574023,0.6001860462129116,0.7737955089658499,0.6082593854493461,400,0.05605280064046383,0.08463487401604652,0.15578170120716095,0.09744173836894333,400,9.221808462961413,12.772497303026139,14.162725819777252,12.22264222336821,400,0.5763630289350883,0.7982810814391337,0.8851703637360783,0.7639151389605131,400,0.008648406874963287,0.016846141052106793,0.04900140001194352,0.023573000986484353,400,0.1052101933146675,0.13963082995641257,0.2436601676754487,0.15733335208005989,400,0.3595682365847099,0.4199043666711588,0.5212731742804751,0.43222920257873065,400,0.638153783082408,0.6835727865007748,0.7643806054587863,0.6940305720897877,400,0.08098211617503247,0.11552865506524729,0.14210451810033878,0.11326887084730934,400,12.0,13.0,13.0,12.75 +fresh-zh-1,13,14,Q,16,linear_attention,400,0,400,0.3581819446058944,0.6057678542565554,0.9266265729442241,0.6194080411925097,400,0.05562887825071812,0.10121583938598633,0.16555388420820238,0.10790316329570487,400,8.984764348820594,10.993171368600777,12.804795384360892,10.918961714127299,400,0.5615477718012871,0.6870732105375486,0.8002997115225557,0.6824351071329562,400,0.016635458727710436,0.03036327305184136,0.05205285202134545,0.03328436388755628,400,0.1250005151554404,0.16504284241999867,0.2317043184887997,0.1735018718689452,400,0.4214582738396168,0.48794216702592397,0.5581305760766958,0.48918969051415956,400,0.6920941428310349,0.7540016609510316,0.8118286564254139,0.75251594489739,400,0.05127560458582482,0.08648351968154439,0.12220059621684103,0.0871981589591026,400,11.0,12.0,13.0,11.9875 +fresh-zh-1,13,14,K,16,linear_attention,400,0,400,0.3721212899079547,0.6305268344003707,0.9718979531899096,0.6449294262027252,400,0.05363687835633755,0.09777480736374855,0.1829571843147278,0.1116250408673659,400,8.98737507106024,11.212097542070406,13.059731885315017,11.083351780568323,400,0.561710941941265,0.7007560963794004,0.8162332428321886,0.6927094862855202,400,0.015009336843665294,0.02846866968756794,0.05201834647142469,0.031837631525605606,400,0.12308597876710539,0.16251156497169478,0.23243101965169033,0.1712443740005486,400,0.4164322115449284,0.48113854088534125,0.5536128445704002,0.48438083221055267,400,0.6837956313094269,0.7472076678467134,0.8051837376370223,0.7453305523637757,400,0.06012654147533741,0.09088036749332673,0.12781760922792534,0.09280544570777137,400,11.0,12.0,13.0,12.175 +fresh-zh-1,13,14,V,16,linear_attention,400,0,400,0.29954221642110496,0.4848028526175767,0.9077605601400137,0.5606348308987799,400,0.043849870562553406,0.08329145982861519,0.1724206700921059,0.1018496751692146,400,8.601624836963923,10.945460021931986,13.039453404274523,10.85560632719913,400,0.5376015523102452,0.6840912513707491,0.8149658377671577,0.6784753954499456,400,0.015136398331273958,0.030786349912227617,0.05734093585275203,0.03566599715019743,400,0.11954870810478686,0.16362467289990384,0.24502782207819693,0.1769241861836662,400,0.4124458202716032,0.4875960285747094,0.5675203640802442,0.49039862284247,400,0.6847232716758535,0.7569398418201212,0.8156498747113647,0.7524608350138405,400,0.0550192637263923,0.08342689632152683,0.12084799344469732,0.08657916371396172,400,11.0,12.0,13.0,11.9225 +fresh-zh-1,13,14,QKV,16,linear_attention,400,0,400,0.4210604142397642,0.6132193456869572,0.9099864074960354,0.6461962532845792,400,0.06235298551619053,0.09698081389069557,0.1679705634713173,0.10688274818472564,400,9.559044451771125,11.462648813151414,13.229307283707383,11.372536790697668,400,0.5974402782356953,0.7164155508219634,0.8268317052317115,0.7107835494186042,400,0.013962413468451337,0.026389219655446716,0.044920498319119116,0.02927729497894551,400,0.12302451292251601,0.15709634512624568,0.2172133642117814,0.16492861000588327,400,0.4117132816521857,0.4697141174747012,0.5382139792223727,0.4737089325042268,400,0.6784471857426405,0.7417767439401142,0.7971131902854659,0.7392774549840461,400,0.06311006935352877,0.09444935201900284,0.13123542279990064,0.09541277970078714,400,11.0,12.0,13.0,12.27 +fresh-zh-1,14,15,Q,16,full_attention,200,0,200,0.6577422137372195,0.9171648286283016,1.1852634995244442,0.9181472365325317,200,0.11345553621649743,0.15000705420970917,0.1810215711593628,0.15020154509693384,200,9.974972828525296,10.98748633023548,12.015739261979009,11.008383435577516,200,0.623435801782831,0.6867178956397175,0.7509837038736881,0.6880239647235947,200,0.022105795807046344,0.03041346995763028,0.040267832978357676,0.030734119759068097,200,0.13690393024083136,0.16355526124714392,0.19754851941285856,0.16622357418701442,200,0.45715092057926165,0.49559856760163823,0.5399387709213074,0.49583705303041015,200,0.7112572552939894,0.7405481171619599,0.7795045856523062,0.7424820054653612,200,0.0671746009092024,0.09349980356859358,0.10992506218390213,0.09149286381927257,200,11.0,12.0,13.0,12.19 +fresh-zh-1,14,15,K,16,full_attention,200,0,200,0.4558223349275068,0.661193341948092,0.928560869814828,0.6845277983671986,200,0.09089065045118332,0.14050742238759995,0.2652635604143142,0.16330237239599227,200,6.517073625860159,8.448169533418689,10.216135117656798,8.420845466774376,200,0.40731710161625995,0.528010595838668,0.6385084448535499,0.5263028416733985,200,0.037743533953733324,0.059597051779429054,0.09700626283126307,0.06391106362676019,200,0.16545158667610715,0.21786337825417829,0.3109351348229441,0.23088798633735585,200,0.5273401415002065,0.5891054982550972,0.670537313858403,0.5982072906277595,200,0.7789887782313337,0.8273318255285389,0.867367099825448,0.826735536039555,200,0.03556134799369811,0.05069459324987072,0.06732191690401192,0.05158814468967845,200,10.0,11.0,11.099999999999994,10.54 +fresh-zh-1,14,15,V,16,full_attention,200,0,200,0.39862508603837343,0.553866085014306,0.8149181741755454,0.5910873380908742,200,0.07487345486879349,0.1185121051967144,0.26099399924278255,0.1435592506825924,200,6.23539240573393,8.575625033151486,10.542810464490822,8.593766850785837,200,0.38971202535837063,0.5359765645719679,0.6589256540306764,0.5371104281741148,200,0.03452489179458992,0.05771847128863582,0.10441198167052927,0.06257753538089733,200,0.17394930222282973,0.22248368839866833,0.3403276449735198,0.23622923989761208,200,0.5139030605907748,0.5745139919281239,0.6391214979272684,0.5741994356844878,200,0.7497449375485372,0.7985724668806748,0.8343295262531546,0.7939948756911482,200,0.047876148136253904,0.06173908973538601,0.09238906906297285,0.06512158069667205,200,11.0,11.0,12.0,11.26 +fresh-zh-1,14,15,QKV,16,full_attention,200,0,200,0.6458288012538105,0.7748749549500644,0.9652682330459356,0.7980924565205351,200,0.11018678247928619,0.13081123679876328,0.18139647394418718,0.14203495394438506,200,8.73400421523041,10.246712388513933,11.44923822507546,10.193826057069092,200,0.5458752634519006,0.6404195242821208,0.7155773890672162,0.6371141285668183,200,0.02649819570006886,0.03743176517140752,0.05546138875430876,0.03928103644010278,200,0.1449581241047069,0.17280844858768796,0.22752581143256917,0.179581028414308,200,0.47710706891360144,0.5362226771385954,0.5856630329020351,0.531838099732763,200,0.7295533715428937,0.7722563931895174,0.8144699553256052,0.7730578378245819,200,0.05475231118994709,0.07820279139223463,0.09810128173046476,0.0765137326843442,200,11.0,12.0,12.0,11.695 +fresh-zh-1,14,15,JS,16,full_attention,200,0,200,0.019945300565086654,0.03742143244858198,0.06954158965454553,0.04249801874332888,200,0.006600805884227156,0.013597384095191956,0.0299729011952877,0.01775519482791424,200,2.723727922639179,4.419071740960671,6.453804065895095,4.506546875024986,200,0.1702329951649487,0.27619198381004195,0.4033627541184434,0.2816591796890616,200,0.09861066336700716,0.17471540683448422,0.3249596363003885,0.19923313577665447,200,0.25891546649056973,0.36224561549091905,0.5691559297756097,0.3955265302614648,200,0.6979655399278633,0.8148032333471716,0.9034887493058987,0.802466854880239,200,0.9009231053331983,0.948978783177368,0.9757591574499868,0.9435997776413383,200,0.005237995813748987,0.012665228570550776,0.02465008873120709,0.014240310710541213,200,4.0,6.0,8.0,6.415 +fresh-zh-1,14,15,sqrtJS,16,full_attention,200,0,200,0.43292285297065974,0.5795048493891954,0.8028445993550122,0.598374522572849,200,0.08124532625079155,0.11660745367407799,0.17312516421079635,0.1251748430915177,200,7.27187131575747,9.280908514420442,11.168043529324263,9.19956206262213,200,0.45449195723484187,0.5800567821512777,0.6980027205827665,0.5749726289138831,200,0.028844266270079803,0.04826461737077646,0.08001845356029015,0.052541086242545665,200,0.1521058685250012,0.20071546433315057,0.2787634172105522,0.20895935456315512,200,0.48676783085182757,0.5606942171838359,0.6427062218665469,0.5640766258320425,200,0.7445924303680195,0.799933686349982,0.850776678718176,0.7998207023229159,200,0.04972880916576016,0.07123257799778379,0.09336855652866997,0.07216344075869018,200,10.0,11.0,12.0,11.335 +fresh-zh-1,15,16,Q,8,linear_attention,400,0,400,0.19986069407314064,0.46389445289969444,0.8956709902733565,0.5090842917381088,400,0.04299015626311302,0.10527447238564491,0.2599944949150086,0.12798810095293447,400,5.834645955809603,6.856790631430793,7.407216043831221,6.73552036533139,400,0.7293307444762004,0.8570988289288491,0.9259020054789027,0.8419400456664238,400,0.011432559850753476,0.023818087522427103,0.053017618729649545,0.028307218148329493,400,0.18324670830013334,0.22686992103152165,0.31176969445782493,0.2387361114995067,400,0.3449582984207271,0.3928509719419186,0.4596047972057098,0.39629894065743654,400,0.6058274989044762,0.6498933257271503,0.6973570857052434,0.6514275959246021,400,0.12112850999103765,0.14790487657338322,0.17462526367807285,0.14802789306899325,400,7.0,7.0,7.0,6.995 +fresh-zh-1,15,16,K,8,linear_attention,400,0,400,0.2865625012665987,0.46163270249962807,0.7159958682954312,0.48214415930677207,400,0.05249631702899933,0.09312321618199348,0.1700477421283722,0.10407672118395567,400,6.513632666676123,7.123124686724383,7.498416961966744,7.058480003951019,400,0.8142040833345153,0.8903905858405479,0.937302120245843,0.8823100004938774,400,0.009555980058244042,0.01758609900382032,0.03259904318889367,0.019555700837432296,400,0.17557890438178544,0.2085092995367986,0.2516834065681224,0.21139714156376396,400,0.33537148359147667,0.37606616382062896,0.42800448153384124,0.3787479168038449,400,0.6022690095489,0.6364912429900353,0.6810532471069338,0.6387765209735242,400,0.12892438858843056,0.15751327181530345,0.1825052865260191,0.156796619122968,400,7.0,7.0,7.0,7.0025 +fresh-zh-1,15,16,V,8,linear_attention,400,0,400,0.3101591121405363,0.414548727683723,0.6638900108635426,0.4536746086040512,400,0.062248042225837706,0.08890051767230034,0.1481402471661568,0.09876976327970624,400,6.375527351263597,6.946734358280413,7.392494910423758,6.901467936472129,400,0.7969409189079496,0.8683417947850516,0.9240618638029697,0.8626834920590162,400,0.011739804231719175,0.021660037320278387,0.0363999217804117,0.023460530141269996,400,0.1773245538495785,0.21211547276437873,0.25762269090809325,0.21560968323067572,400,0.34178296020310805,0.3867883582709063,0.4354503137200121,0.38860321891175004,400,0.6159145676009288,0.6565733808317405,0.6972125027654101,0.6558877640936979,400,0.10916973944482057,0.1414943598450648,0.1716854432779864,0.14130149071505607,400,7.0,7.0,7.0,6.96 +fresh-zh-1,15,16,QKV,8,linear_attention,400,0,400,0.3314094701781869,0.4764906894415617,0.7405438203364613,0.5044400906306692,400,0.0644989974796772,0.0975835919380188,0.19508592635393146,0.11217128470540047,400,6.461353840637956,7.098616509554886,7.474204052423043,7.030282980550095,400,0.8076692300797444,0.8873270636943608,0.9342755065528804,0.8787853725687619,400,0.010049726857083547,0.01814002479521407,0.03401873707408182,0.02020727729036347,400,0.17998469835815162,0.21026860057469182,0.2675367136732707,0.21648070657757884,400,0.3397267825731491,0.3770363418833491,0.4256815960359429,0.3793530208424864,400,0.6027215378027868,0.6376038756387015,0.6753351810058252,0.6385888219285999,400,0.13112802890263664,0.15820078198535004,0.1813363088607309,0.15757380853151823,400,7.0,7.0,7.0,7.0075 +fresh-zh-1,16,17,Q,16,linear_attention,400,0,400,0.08711719287675807,0.2994851019466296,0.573016984621063,0.32165666329921805,400,0.01310598086565733,0.04576076939702034,0.09498987272381783,0.05223502334440127,400,9.583010174830612,11.840225729216735,13.324799122374221,11.582809323374299,400,0.5989381359269133,0.740014108076046,0.8327999451483888,0.7239255827108937,400,0.013384581465064207,0.02342176432204445,0.04464143732069111,0.02759057673775662,400,0.11746223739112596,0.15123771411546127,0.21382206660995698,0.1620249147955738,400,0.4028018966273583,0.45963470146711133,0.5354911437966935,0.464178923219749,400,0.676133827954727,0.7253601585712695,0.7811042705078216,0.726551221455319,400,0.07287342136913855,0.09538573412747754,0.12436979498279221,0.09748222541172494,400,12.0,12.0,13.0,12.3775 +fresh-zh-1,16,17,K,16,linear_attention,400,0,400,0.3286965986713767,0.4223298658616841,0.5934893887024373,0.4472175025119213,400,0.04229480139911175,0.059895411133766174,0.09417680427432061,0.06480434986762702,400,10.693028184724891,12.383167510963343,13.468265855749474,12.219783722162196,400,0.6683142615453057,0.7739479694352089,0.8417666159843421,0.7637364826351373,400,0.01253184928229949,0.019471808687132178,0.033086804079425956,0.0214448347101134,400,0.11219071860921055,0.13757776612288575,0.18140014587027972,0.14428989032577394,400,0.395864530231225,0.4362149194990499,0.49761317840940616,0.4428913856845753,400,0.6670044234864881,0.7081641271470507,0.7580535919767092,0.7106260600031876,400,0.08218487972765465,0.10538948020512309,0.12561972850394257,0.1048475434681044,400,12.0,13.0,13.0,12.6175 +fresh-zh-1,16,17,V,16,linear_attention,400,0,400,0.29818206932395697,0.4106921355705708,0.5565018072258681,0.4215392938845616,400,0.04070471748709679,0.060807785019278526,0.1028570704162121,0.06672028207220138,400,9.495061325427482,11.908249339816994,13.223724295115607,11.602071430960843,400,0.5934413328392176,0.7442655837385621,0.8264827684447255,0.7251294644350527,400,0.013996491102012917,0.02290709493773848,0.04567249861168296,0.027394742466076868,400,0.11745855227098129,0.14527100560214212,0.2092226785607464,0.15740708318409813,400,0.40185889257705854,0.4532001243796322,0.5424797312566577,0.4635072268078833,400,0.6805855945888534,0.7287748652829271,0.7975287519249424,0.7326277210266369,400,0.062150436505154893,0.09350211957618837,0.11988088587616197,0.09243153804781819,400,11.0,12.0,13.0,12.15 +fresh-zh-1,16,17,QKV,16,linear_attention,400,0,400,0.3224775362992659,0.4170524077489972,0.5280065821483731,0.4254810506635113,400,0.040970437973737714,0.05731347016990185,0.08391455858945848,0.06086966967675835,400,10.81968177679432,12.499710611648208,13.58935020530476,12.351658313483842,400,0.676230111049645,0.781231913228013,0.8493343878315475,0.7719786445927401,400,0.01182617386760729,0.018668643309448833,0.03191910415011717,0.020449086487372106,400,0.11375288267718857,0.1369316621200216,0.17717501161571683,0.14239705159185362,400,0.392013385518266,0.4328109726063489,0.49936021500857736,0.4394045587806595,400,0.6618637501543095,0.7042037534592438,0.7559459674777604,0.7069288615362622,400,0.08320365562586786,0.10744502171180498,0.12997793867890256,0.10696707120563684,400,12.0,13.0,13.0,12.6625 +fresh-zh-1,17,18,Q,16,linear_attention,400,0,400,0.11444642521673815,0.3289668773068115,0.5647181982640177,0.33337109683678134,400,0.026945790834724904,0.08264965564012527,0.15005922019481663,0.08756069282535464,400,6.0035884150547645,8.928508890559545,10.900737928029212,8.710076795183792,400,0.3752242759409228,0.5580318056599716,0.6812961205018258,0.544379799698987,400,0.031186073629575987,0.052800931177753765,0.11100488581622223,0.06240130384823678,400,0.19199674732986152,0.2490011326242375,0.35883630646701725,0.26310991506586956,400,0.4648490098961464,0.5222855178204693,0.6303564401796478,0.5348779070117441,400,0.7040174974946337,0.7523774476737031,0.8174405510976654,0.7571019525652098,400,0.06628559825730737,0.0912277696545262,0.11763073004175559,0.09082397585155792,400,11.0,12.0,13.0,12.1125 +fresh-zh-1,17,18,K,16,linear_attention,400,0,400,0.17144957436248662,0.30153246014378965,0.42404144285246737,0.3016565287492995,400,0.04597516842186451,0.07409296557307243,0.11804260313510895,0.0783864412875846,400,6.973458341860775,8.69440343611236,10.544563287056361,8.746841117647467,400,0.43584114636629845,0.5434002147570225,0.6590352054410226,0.5466775698529667,400,0.03449132513227807,0.05601762435137562,0.08629459024746584,0.058470424258631054,400,0.20356528501763435,0.26047895141792266,0.32190310711945586,0.2613304243990743,400,0.46886155012263386,0.5267360414592067,0.583467152958784,0.5270573806988736,400,0.7088489808026993,0.7506739046228881,0.7933182284989547,0.7516857318751735,400,0.07120798454805734,0.0951264081130174,0.11640304984400103,0.09463269199708506,400,11.0,12.0,13.0,12.28 +fresh-zh-1,17,18,V,16,linear_attention,400,0,400,0.2788433980429545,0.35655369400046766,0.4507291147019714,0.3619566791538091,400,0.09207431375980377,0.12409733980894089,0.15486080795526505,0.12325305098667741,400,4.566530384065741,6.346931722783292,8.782143948968194,6.484276544655219,400,0.2854081490041088,0.3966832326739558,0.5488839968105121,0.4052672840409512,400,0.054791907387735136,0.10139368017053182,0.1669172264480259,0.10758896086343032,400,0.25107537815491116,0.3429291069978044,0.436681824149149,0.34396526911239717,400,0.5269692736723254,0.6064357640901277,0.6624302106970968,0.6007640487598719,400,0.7461469042457075,0.7982054779231895,0.8417465189807074,0.7964446593033108,400,0.05434727123603426,0.07718751395880855,0.1006514460061799,0.07714848203318231,400,10.0,12.0,13.0,11.5475 +fresh-zh-1,17,18,QKV,16,linear_attention,400,0,400,0.26694579506292937,0.34472838416695595,0.4579548066481948,0.3541167849037447,400,0.07474510371685028,0.1006401963531971,0.13284760862588885,0.10233625747263432,400,6.255063007101469,7.863978606112392,9.494918795352538,7.87329112497948,400,0.3909414379438418,0.4914986628820245,0.5934324247095336,0.4920806953112175,400,0.045674142425202666,0.06897302143369652,0.10386185808151428,0.07241407563426625,400,0.23578694193978422,0.28736456120209564,0.34897081549834713,0.29107630782158267,400,0.4947031268324942,0.5495153297628259,0.6052951389501732,0.550273656553461,400,0.7241112361876384,0.7646267173108904,0.8049597108393459,0.7649382838899986,400,0.06931587054786616,0.08962916571450098,0.10955993811315815,0.08957083073094946,400,11.0,12.0,13.0,12.1325 +fresh-zh-1,18,19,Q,16,full_attention,200,0,200,0.40186612848192454,0.4660476525314152,0.5630262769758702,0.47487199571914973,200,0.0650772623717785,0.07989485561847687,0.10173414945602416,0.08234241669997573,200,10.12810645809385,11.601534563597166,12.725747312893004,11.523018809154651,200,0.6330066536308656,0.7250959102248229,0.7953592070558128,0.7201886755721657,200,0.017152903726017504,0.025275235926569224,0.038651569538112475,0.02658703160192312,200,0.14592901287507962,0.17147407737309045,0.20097218725826207,0.17335960590672828,200,0.42176770824454984,0.46316859440634306,0.5390721155220035,0.47015618742144694,200,0.6737976394302598,0.7086594320210411,0.7619974861112238,0.711631283654674,200,0.08894774717197554,0.11645042191684796,0.14417352219169513,0.1170132078824728,200,12.0,13.0,14.0,12.95 +fresh-zh-1,18,19,K,16,full_attention,200,0,200,0.2383557878434658,0.2800897735869512,0.40347801889292895,0.2980342358537018,200,0.04454663395881653,0.05757962539792061,0.11347527503967285,0.0688006903231144,200,6.659188448098644,9.619182260087715,11.000871731631374,9.0697628004,200,0.41619927800616524,0.6011988912554822,0.6875544832269609,0.566860175025,200,0.030295354552186994,0.044225163876059265,0.09360747781267274,0.05872871124822177,200,0.16919428711946752,0.20354627502050726,0.3202322395846846,0.23121901800390596,200,0.500888998929547,0.5510143276679249,0.6311179375958716,0.5598069054867643,200,0.7390084871905289,0.7650663769812541,0.808768243324379,0.7697533244238843,200,0.07626054912607459,0.08974869381532198,0.10836103910293753,0.09102594870166897,200,11.900000000000002,12.0,13.0,12.14 +fresh-zh-1,18,19,V,16,full_attention,200,0,200,0.2105618339497596,0.26664538076147437,0.6062462885864074,0.32882167091127484,200,0.04057726487517357,0.05638102628290653,0.10011147707700728,0.06374377176165581,200,9.063332882979116,11.018012659841542,12.614013817346505,10.955583720142782,200,0.5664583051861948,0.6886257912400964,0.7883758635841566,0.6847239825089239,200,0.01789635619778122,0.03014528509096625,0.05102403166496306,0.032138560554408696,200,0.15899317303284274,0.1984824956334712,0.2525142685702805,0.20128350024205993,200,0.4243098691093764,0.46811114039326907,0.5247275695931488,0.468130859348047,200,0.6536294591357411,0.6966165360412374,0.7344836899959832,0.6957549332645888,200,0.10778337214586695,0.12635339778205795,0.15696974957914428,0.12776715023867793,200,13.0,13.0,14.0,13.2 +fresh-zh-1,18,19,QKV,16,full_attention,200,0,200,0.3160225347615778,0.37049319688230753,0.5122994367033243,0.3931979568884708,200,0.05125492662191391,0.06661264598369598,0.09244697988033293,0.06970386667177081,200,10.207788622643587,11.45153724767245,12.62791443689083,11.39239019619744,200,0.6379867889152242,0.7157210779795281,0.7892446523056769,0.71202438726234,200,0.017802356626204725,0.02647952857663271,0.037828739337075895,0.027571846487599604,200,0.14179378283656605,0.1789277489871398,0.20475874522368429,0.17752368670248805,200,0.42932173162525245,0.468638335677255,0.5210576323533085,0.4737948927943687,200,0.6715027907704788,0.7155576126236289,0.7461306792649463,0.7130189220379008,200,0.09833837692914749,0.11557345949284488,0.13936113192895164,0.11688660634552829,200,12.0,13.0,13.099999999999994,12.98 +fresh-zh-1,18,19,JS,16,full_attention,200,0,200,0.007393096410646649,0.012181064067817715,0.018965315184050268,0.012800878722067068,200,0.003304202063009143,0.007077381014823914,0.012031846866011619,0.007414750217576511,200,1.8131037119282571,2.5861945507183703,3.8056885226410704,2.773240141124483,200,0.11331898199551607,0.16163715941989815,0.2378555326650669,0.1733275088202802,200,0.21362110907349677,0.3457801242352132,0.5216823690293247,0.35375158226090586,200,0.39054159599782057,0.5554795386289151,0.7298192826476227,0.5607862203853119,200,0.8382733198357065,0.9014582827651687,0.9417105610582329,0.8940616955368718,200,0.9446463243627984,0.9674112908715309,0.9823178519511492,0.964551388545188,200,0.004339754926425057,0.007983271427843467,0.013694931597522919,0.008508104633810631,200,3.0,4.0,6.0,4.64 +fresh-zh-1,18,19,sqrtJS,16,full_attention,200,0,200,0.24335797766689213,0.29331015225034207,0.3536794791696593,0.29521937190263997,200,0.05748218968510628,0.08412716910243034,0.10968977808952332,0.08364754555746913,200,5.711247110272558,7.024246621721719,8.823026151557187,7.196968633935012,200,0.3569529443920349,0.43901541385760745,0.5514391344723242,0.44981053962093825,200,0.054229127537616,0.08518829188617832,0.12009932598436433,0.08591510904237093,200,0.2150525931638637,0.2828655296956779,0.3538350878800364,0.28249004018624446,200,0.5713467217876544,0.6286265816374231,0.6790948191973873,0.6282328187848598,200,0.788729863215255,0.8190756238169092,0.8500392600289038,0.8184562214453764,200,0.0516866276508774,0.06341780080102401,0.07863042920113657,0.06433900282961363,200,10.0,11.0,12.0,11.0 +fresh-zh-1,19,20,Q,8,linear_attention,400,0,400,0.15525705595500766,0.3394973468966782,0.6571132013574245,0.3765427278200514,400,0.04618050307035446,0.09498502314090729,0.2521578311920167,0.12125425999984145,400,3.9186470605530057,5.544321493315213,6.539376941055835,5.442544432059326,400,0.4898308825691257,0.6930401866644016,0.8174221176319794,0.6803180540074157,400,0.031908305681334834,0.06327399320957156,0.14878890452186816,0.07481138653408684,400,0.23105027707205306,0.296195527567483,0.4293521309738975,0.3097218065751807,400,0.41221028593805686,0.47955417790287896,0.6106232802894005,0.49487810267112964,400,0.690652918222389,0.7580765843999429,0.8385641110030124,0.7615882490962289,400,0.04741852897855421,0.0849469426155012,0.11846163443323997,0.08455097397071672,400,5.0,6.0,7.0,6.16 +fresh-zh-1,19,20,K,8,linear_attention,400,0,400,0.2675408917479217,0.3827636414207518,0.6346319142729044,0.4226680134201888,400,0.0657367616891861,0.1260973885655403,0.24239765256643298,0.13847129544243217,400,4.167757318121759,5.298839034982416,6.479710135864358,5.307034668169767,400,0.5209696647652199,0.662354879372802,0.8099637669830447,0.6633793335212209,400,0.03351759089842679,0.0728235627895297,0.1313568212441941,0.07894804102788361,400,0.2200209713108122,0.31088624049972713,0.41728173071230773,0.3157428005653104,400,0.4095813956147669,0.5005013303139483,0.5839266861723282,0.4984212081315583,400,0.7062365762020371,0.777583468507881,0.8367804181475219,0.7739838892164036,400,0.052986015112582226,0.07733663049086614,0.11218255577304988,0.08043071935381756,400,6.0,6.0,7.0,6.105 +fresh-zh-1,19,20,V,8,linear_attention,400,0,400,0.2774194669444114,0.36462423647753894,0.5320786954835058,0.38472252163453957,400,0.0807145245373249,0.10361829400062561,0.14211224466562272,0.10949894278310239,400,4.7007518590285855,5.4682866211277705,6.285478487002955,5.484944301562345,400,0.5875939823785732,0.6835358276409713,0.7856848108753693,0.6856180376952932,400,0.038967917005168114,0.06614016003006074,0.10026506092248094,0.06847465666403775,400,0.23000650050593804,0.27872310557779834,0.3504997719943814,0.28806588638896147,400,0.433386446056023,0.49836376683005823,0.5636916956406838,0.4961686099239049,400,0.7065554738300994,0.7747459772120875,0.8241917384706339,0.771155672700703,400,0.053060863002019315,0.08009137903232538,0.11597468426977454,0.08279114099653687,400,6.0,6.0,7.0,6.1675 +fresh-zh-1,19,20,QKV,8,linear_attention,400,0,400,0.29529009452089666,0.40087789110839367,0.5732318122871221,0.41862270907731725,400,0.08025466576218605,0.11346953362226486,0.20841896235942844,0.13092674260027706,400,4.446845475571162,5.488597445848338,6.3884772953095,5.437088566588267,400,0.5558556844463952,0.6860746807310423,0.7985596619136875,0.6796360708235334,400,0.03603647820786743,0.06536688384837219,0.1141468734730059,0.0719203431196204,400,0.23886409419423785,0.2982476099626413,0.3924174480819068,0.30625292162201306,400,0.425356881747965,0.49118959834060877,0.570381516538761,0.4951553398055028,400,0.7006763525417299,0.7691835719259329,0.8289667192633471,0.767764499128189,400,0.057491969063298315,0.08305301189599479,0.11418334940728253,0.08372321816680739,400,6.0,6.0,7.0,6.1425 +fresh-zh-1,20,21,Q,16,linear_attention,400,0,400,0.09100260541308672,0.1578803969314322,0.47679477347992394,0.21357492956944044,400,0.013946398627012968,0.0253577483817935,0.09228626117110254,0.038190908087417486,400,9.669408779086732,11.415225398347491,12.73911800259806,11.269053017692489,400,0.6043380486929207,0.7134515873967182,0.7961948751623787,0.7043158136057805,400,0.017064928782645285,0.026775793622037178,0.043646979492003656,0.029352397961489664,400,0.13165795663444885,0.16089411479034343,0.21621911319216935,0.1687203318248561,400,0.4244900924599479,0.4785433290639434,0.5435475391333369,0.48244030229373797,400,0.6891106183362785,0.7271872743655241,0.7826357888469359,0.7329405217535465,400,0.07527590514952104,0.10159258586743489,0.12333446763120781,0.10071467690465907,400,12.0,13.0,13.0,12.4725 +fresh-zh-1,20,21,K,16,linear_attention,400,0,400,0.16491668410599233,0.30600598198361695,0.41316689671948553,0.31079191313765475,400,0.02744064182043076,0.051021795719861984,0.07823709771037102,0.05460934706497937,400,9.020126109399264,11.148648245596211,12.396907423978336,10.890723514007611,400,0.563757881837454,0.6967905153497632,0.774806713998646,0.6806702196254757,400,0.019376325175393984,0.0290101052022511,0.05158780053782289,0.03347859885421094,400,0.13250327659397218,0.16390620321480015,0.24047471914384613,0.1774524418725001,400,0.4426878812478206,0.49426622642259455,0.5536032459377827,0.4962049015508903,400,0.7019534765744023,0.7371161884786097,0.7748532936346252,0.737450143719988,400,0.08177054256171805,0.100617967890347,0.11922358240559534,0.10029121070242926,400,12.0,13.0,13.0,12.4925 +fresh-zh-1,20,21,V,16,linear_attention,400,0,400,0.22574676314834505,0.30166250735055655,0.448491025576368,0.32166607168008343,400,0.03758852705359459,0.055542927235364914,0.10080730840563774,0.06360128117259592,400,7.279698708674357,10.685878426636492,12.512606164122632,10.26985391428973,400,0.4549811692921473,0.6678674016647808,0.7820378852576645,0.6418658696431081,400,0.018580699273323027,0.033153549076843286,0.07986051152476253,0.04224451734680011,400,0.1387595197298212,0.18047811741841496,0.2993860884823669,0.19764674845668595,400,0.43731306360086736,0.5060429621407943,0.6030909455895996,0.5146613164609688,400,0.7004034266372262,0.7476760187316618,0.802937517172612,0.7495566218730425,400,0.06760458271448931,0.09325404642082735,0.11711260865480333,0.09247357440744867,400,11.0,12.0,13.0,12.205 +fresh-zh-1,20,21,QKV,16,linear_attention,400,0,400,0.21518497793003916,0.28149201755877584,0.41662318254821007,0.3026864863996161,400,0.037530127167701724,0.05159826576709747,0.08380187228322031,0.056122346417978405,400,8.578341197215607,10.74834736709455,12.49694583917458,10.63620047415439,400,0.5361463248259755,0.6717717104434093,0.7810591149484113,0.6647625296346493,400,0.01868752153110747,0.03257339644350163,0.05767749774725549,0.03574639009449255,400,0.1381482033714521,0.17714372896328257,0.2574492118931321,0.18849974620724885,400,0.4419836057510083,0.5033089776957218,0.559189490797357,0.5017523334597588,400,0.7010759845319436,0.737958688161038,0.7742489859483731,0.7385488075890002,400,0.08046029679321556,0.09913061480170313,0.11849481028389905,0.0992009448967302,400,12.0,12.0,13.0,12.465 +fresh-zh-1,21,22,Q,16,linear_attention,400,0,400,0.0638073873706162,0.15770221455022693,0.3229407866485417,0.17402296393411235,400,0.00827278159558773,0.020439019426703453,0.05504508726298809,0.030880103794625028,400,10.956545395669451,13.009093934470757,14.146357049170428,12.553184850616509,400,0.6847840872293407,0.8130683709044223,0.8841473155731517,0.7845740531635318,400,0.008735550152923651,0.015327467139166084,0.03068794874960493,0.022075422490765037,400,0.1103447908992042,0.1377652289869727,0.19578295029911008,0.15332308972371,400,0.37227052823390233,0.4192664246088049,0.48966976594113054,0.42861407325127365,400,0.6317030284209187,0.6776187750204767,0.7301179426123745,0.6820589771341953,400,0.10099390697790879,0.1268696856947495,0.15536662207298013,0.12783967131998378,400,13.0,13.0,14.0,13.1375 +fresh-zh-1,21,22,K,16,linear_attention,400,0,400,0.09741592470090837,0.21235808753408492,0.33288378436118365,0.21420880685123847,400,0.014754499867558482,0.028414766304194927,0.04797113463282587,0.030325422501191496,400,11.277069871004235,13.054859308406666,14.179173104085924,12.874476857178797,400,0.7048168669377647,0.8159287067754166,0.8861983190053703,0.8046548035736748,400,0.0085610426039156,0.015039831361483035,0.02792068290852134,0.016884931462583762,400,0.10611469307035479,0.14159071334269058,0.19275314545378658,0.14519345661898928,400,0.3707465013911077,0.41039248686792795,0.46615928867570466,0.415185903909173,400,0.6374763132555936,0.6724219935445845,0.7138607505386906,0.6745149800677268,400,0.10904332055266763,0.13059542805893948,0.1530590140265871,0.13054673841616418,400,13.0,13.0,14.0,13.235 +fresh-zh-1,21,22,V,16,linear_attention,400,0,400,0.13401092891581357,0.1944151925854385,0.31463873228058226,0.2110497849449166,400,0.017851484939455986,0.030416744761168957,0.05377412773668766,0.03353422887157649,400,9.019782820844243,12.831819279487862,13.98700610826198,12.223251618279864,400,0.5637364263027652,0.8019887049679914,0.8741878817663737,0.7639532261424915,400,0.009594590783200348,0.016460024775304644,0.051591986613178355,0.024231169072713613,400,0.11298294130091135,0.14360079074992677,0.2599274339240808,0.1634574645587918,400,0.3753465582658847,0.4227827968976451,0.5114027864098041,0.4340739866154702,400,0.6420315240429671,0.6826627550401714,0.7386390454552453,0.6853793198154758,400,0.10166157257753133,0.12752871192834653,0.14951535898715418,0.12694117917653777,400,13.0,13.0,14.0,13.1525 +fresh-zh-1,21,22,QKV,16,linear_attention,400,0,400,0.1325829988112673,0.209606354823336,0.2849590593017638,0.2128292384831002,400,0.018658262863755225,0.029178920201957226,0.04782337360084058,0.03356043431442231,400,10.755941306123653,12.985362229166007,14.109065675052369,12.639862789920217,400,0.6722463316327283,0.8115851393228755,0.881816604690773,0.7899914243700136,400,0.008934854082441372,0.015477107845314886,0.032503459699744705,0.019509789353500794,400,0.10981471305764977,0.14379035327591175,0.21108071221739746,0.1537745280956764,400,0.3727250764172539,0.4157947406682161,0.4789287918367615,0.4210803922181682,400,0.6348954132432477,0.6751433278233427,0.7157782995894862,0.675016521542984,400,0.11033907564139822,0.1321006207445523,0.15223692759961058,0.1316575972431657,400,13.0,13.0,14.0,13.2275 +fresh-zh-1,22,23,Q,16,full_attention,200,0,200,0.3026728035882115,0.37655932281631976,0.46557707500178364,0.3813775059761247,200,0.04963986799120903,0.0744844377040863,0.12148832082748412,0.07976769370958209,200,8.158460086767704,9.68046214852427,11.099525128742243,9.653566909599014,200,0.5099037554229815,0.6050288842827669,0.6937203205463902,0.6033479318499384,200,0.029433813967526593,0.043520912202843494,0.06408177446837236,0.0460129208423802,200,0.15447485364824012,0.19627197084865178,0.2737614323565545,0.20513040499060614,200,0.48598316241886763,0.5392094267902581,0.5846335035116358,0.5344260250513373,200,0.7474379347931019,0.7802175695463072,0.8053826026015454,0.7768267640960187,200,0.06112540760596603,0.07497044494860407,0.09211129176370675,0.07563308191429308,200,11.0,12.0,12.0,11.69 +fresh-zh-1,22,23,K,16,full_attention,200,0,200,0.28945007940055806,0.36516114440746605,0.48591423322213806,0.3753078854223713,200,0.07137070298194886,0.12555516883730888,0.15731682032346725,0.12020358331501484,200,5.059222337631255,6.533838025570682,8.445334108931446,6.731609824608861,200,0.3162013961019534,0.4083648765981676,0.5278333818082154,0.4207256140380538,200,0.05964055496211359,0.09658723733189026,0.14416951786206372,0.09757853452436181,200,0.22990450380723995,0.3263121331322245,0.4043384709146647,0.3189793342842524,200,0.5805636112578474,0.6106102178790714,0.6557902952821408,0.6143969532199712,200,0.7985027454923859,0.8203908734288934,0.846657177240993,0.8196459190867469,200,0.04985520469823094,0.057907431236357276,0.06828933856993738,0.05885155268220192,200,10.0,11.0,11.0,10.88 +fresh-zh-1,22,23,V,16,full_attention,200,0,200,0.19312360133044423,0.246892538969405,0.36598558727419,0.27202045162790456,200,0.043266743421554565,0.07038240879774094,0.17390640228986737,0.09952279672026634,200,3.471213997602875,6.172750443539957,8.14999768688345,5.965572739446648,200,0.2169508748501797,0.38579690272124734,0.5093748554302157,0.3728482962154155,200,0.06421659706818393,0.10621906278958904,0.24062371185341158,0.13112148671853785,200,0.22895501300023957,0.31676578841721836,0.5133794999974186,0.3454599512715491,200,0.5866180593269568,0.6757693701986704,0.7581830887337887,0.6756303261227171,200,0.8183041818037008,0.8634762851639095,0.902966208773753,0.8598603619579761,200,0.030296623242236743,0.04222130957906105,0.058479993506724774,0.0441813352465796,200,8.0,10.0,11.0,9.7 +fresh-zh-1,22,23,QKV,16,full_attention,200,0,200,0.29037469587055964,0.36007841280661523,0.42876356788910924,0.3618773313559359,200,0.06149344183504582,0.10106398910284042,0.14472270011901855,0.10212308172136546,200,5.839062505612701,7.612101863939762,9.64438690124938,7.665156123130973,200,0.3649414066007938,0.47575636649623515,0.6027741813280862,0.4790722576956858,200,0.043933354259961456,0.07346269657826368,0.11601131881578822,0.0775862528069393,200,0.18957159455392047,0.2814065255030375,0.36074185492119154,0.2785400452429898,200,0.5423959525219761,0.5951639662416559,0.6429841849355784,0.5942475007152278,200,0.775967739021575,0.8089392917502702,0.8332560576910509,0.8069472692661399,200,0.05220282240400577,0.06465804028412984,0.07959129731591676,0.0650774446869007,200,10.0,11.0,12.0,11.125 +fresh-zh-1,22,23,JS,16,full_attention,200,0,200,0.0006433107142544969,0.005029468058637576,0.009329614385455896,0.004928803411408012,200,0.0003166275477269665,0.0024420906556770205,0.005078282160684466,0.002524947199417511,200,2.1931030652823855,2.9386259925524554,4.010336386724875,3.0725923688604064,200,0.1370689415801491,0.18366412453452846,0.2506460241703047,0.1920370230537754,200,0.19932094657634944,0.2963147732831179,0.41971051938784687,0.3026397028759481,200,0.36362632817018486,0.49651409730384194,0.6510295296725981,0.5048326123672355,200,0.842685359961803,0.9076509547744371,0.9508851429048261,0.9008439566152725,200,0.9593154414306101,0.978183274274075,0.9915364427157294,0.976535529056014,200,0.0014586561152327788,0.004169070780395262,0.008220400394240034,0.00460339436229952,200,3.0,4.0,6.0,4.405 +fresh-zh-1,22,23,sqrtJS,16,full_attention,200,0,200,0.0633149473782396,0.19330154557246715,0.24875540377106517,0.17162828117536263,200,0.01779393944889307,0.049417492002248764,0.07126184999942779,0.04604077075608075,200,5.629146379436792,6.977966572711463,8.279055098310208,6.967945352487096,200,0.3518216487147995,0.4361229107944664,0.517440943644388,0.4354965845304435,200,0.0621725285869526,0.08619551993155491,0.12282329129553866,0.09071510895108638,200,0.21480057422104684,0.266990922186803,0.33301653823564925,0.2719936755675187,200,0.594021704609691,0.6606513508815572,0.7369617105159236,0.6616751262489386,200,0.8178922005537227,0.8545975415872406,0.8995344632204977,0.8573324619253084,200,0.028710226356380875,0.045251362467490786,0.06104690403816136,0.04556362815948265,200,8.900000000000002,10.0,11.0,9.84 +fresh-zh-2,0,1,Q,16,linear_attention,400,0,400,2.6784210954792798,3.559671798720956,4.315633611753583,3.5250763838225976,400,0.5066291093826294,0.8996946215629578,1.2882700085639953,0.9066566143929958,400,6.117784163448662,8.531549190505725,11.52523814966956,8.692783656485016,400,0.3823615102155414,0.5332218244066078,0.7203273843543475,0.5432989785303135,400,0.025883854945909796,0.0583594992989115,0.10768844080187066,0.06356425468728899,400,0.17041793926498347,0.2506593742694412,0.34858194131701103,0.2561005173416817,400,0.46212054078992065,0.5455619304454083,0.6414843885270962,0.5487701210297806,400,0.7046807235140854,0.760751569011072,0.8199049347590902,0.7616548586967525,400,0.05685607280081057,0.08315166280784761,0.11767255509827633,0.08551447930801687,400,11.0,12.0,13.0,11.96 +fresh-zh-2,0,1,K,16,linear_attention,400,0,400,2.8104895094409588,3.5754500501789153,4.57786109931767,3.656229769074125,400,0.4610318154096603,0.7450397610664368,1.0994075536727907,0.7688645838201046,400,7.512460427712821,10.198398707866057,12.337593432884395,10.00822649513756,400,0.4695287767320513,0.6373999192416285,0.7710995895552747,0.6255141559460975,400,0.01978995926200752,0.03792491821934292,0.07531982691953788,0.04657028923944429,400,0.1448736630137827,0.20618113365045948,0.2840991748227493,0.21291491356964432,400,0.42616791722557923,0.4921625882011538,0.6339759058455573,0.5127754019403266,400,0.6881941121690442,0.7331072229269654,0.8122441042502336,0.7408280859553779,400,0.06357450075704596,0.09070539653501303,0.11907192520056681,0.09055638590408517,400,11.0,12.0,13.0,12.15 +fresh-zh-2,0,1,V,16,linear_attention,400,0,400,1.7271244302392006,2.8793379687704146,4.66863455651328,3.112373010565352,400,0.31386266350746156,0.5519219636917114,1.226453268527985,0.6850567919015884,400,6.887264222471189,9.590149649070376,12.234378218987263,9.601490529650256,400,0.4304540139044493,0.5993843530668985,0.764648638686704,0.600093158103141,400,0.020519398207496854,0.04455889994104878,0.0882087670427868,0.050254306626077334,400,0.1416716918413607,0.20099311928413746,0.30513203078579665,0.21453476691412351,400,0.44239288433326873,0.5266291836488135,0.6289745496130613,0.5315226712406913,400,0.7033953095035427,0.7633178057225993,0.835334681225075,0.7669896969176005,400,0.046942508574499074,0.07595654007959489,0.10908925938061115,0.07703285796184914,400,10.0,12.0,13.0,11.6975 +fresh-zh-2,0,1,QKV,16,linear_attention,400,0,400,2.936536581069231,3.5546105480752885,4.395965920574963,3.630937969654915,400,0.513794070482254,0.780226081609726,1.06846444606781,0.7955444942414761,400,7.72129670134044,9.72016954485241,11.976709803903054,9.76546950612606,400,0.4825810438337775,0.6075105965532757,0.7485443627439409,0.6103418441328787,400,0.022395094374461987,0.043070791464694225,0.07147951253261786,0.045519170640060676,400,0.15839920384162673,0.2159698285858564,0.27867393170202004,0.21805562582437601,400,0.4451752590841819,0.5099094012078482,0.6068989986163271,0.5196597331043674,400,0.6958783874455235,0.7395649248171764,0.7960470134474323,0.7432649491018253,400,0.06336794935501554,0.09091149295795725,0.12208358613776556,0.09267042081655948,400,11.0,12.0,13.0,12.245 +fresh-zh-2,1,2,Q,16,linear_attention,400,0,400,0.5336477428965737,1.1550053376704454,1.755918895499781,1.1622334863040304,400,0.09361197426915169,0.22978638857603073,0.3899625211954117,0.2411746523808688,400,6.781960634979552,9.160391142070782,11.534785610826267,9.154723377705,400,0.423872539686222,0.5725244463794239,0.7209241006766417,0.5721702111065625,400,0.025807365855277584,0.04977719435355327,0.09061331842854252,0.05465675511578203,400,0.1453941230061744,0.1981280887166446,0.27889947302954526,0.20762903478758676,400,0.4621582441523873,0.5568666011738603,0.6516005248551898,0.558283458568097,400,0.7423182282300017,0.815959338378648,0.9074810834063154,0.8194407747783609,400,0.01602720402645506,0.050190706459719625,0.07958469801936839,0.049384819145551796,400,8.0,11.0,12.0,10.4075 +fresh-zh-2,1,2,K,16,linear_attention,400,0,400,0.8472512579988688,1.294328612391837,1.9201772655826064,1.3734041296314898,400,0.1511938065290451,0.25237904489040375,0.41002078950405124,0.2789111100509763,400,7.145554187312737,9.611009828722462,11.556125661195967,9.455693044915314,400,0.44659713670704604,0.6006881142951539,0.7222578538247479,0.5909808153072071,400,0.02563647179252993,0.044317181389924715,0.08261034956951013,0.051438474782410495,400,0.1421917181857805,0.191851669561938,0.27188768816794995,0.20404310070716733,400,0.4691866466938301,0.5389448290915154,0.6301662781434917,0.5439430922482749,400,0.7379483633745343,0.7999119669902908,0.8820116831740423,0.8068884121442349,400,0.0230436059321496,0.05616199994453132,0.08934587416077841,0.055449739808680315,400,9.0,11.0,12.0,10.7 +fresh-zh-2,1,2,V,16,linear_attention,400,0,400,0.5713463072082959,1.074984957871493,1.8424682033248245,1.1459443381384882,400,0.106293685734272,0.21141317486763,0.4074859321117401,0.2377524233609438,400,6.479088462567195,9.273684096203777,11.585670975609442,9.110026463189234,400,0.4049430289104497,0.579605256012736,0.7241044359755902,0.5693766539493271,400,0.025401335531024617,0.04835414519604584,0.09796633830251174,0.05689435407531313,400,0.14184550662475784,0.19020731694753412,0.2985819614374231,0.2081330357637579,400,0.4633980745345449,0.5526392620785863,0.6597568239227333,0.5602812356492427,400,0.7418463091229367,0.8236638617793975,0.912284932627027,0.826010512006913,400,0.015381387134538252,0.046092015472826715,0.08479855267493548,0.04851076992061586,400,8.0,10.0,12.0,10.2375 +fresh-zh-2,1,2,QKV,16,linear_attention,400,0,400,0.802663676487282,1.2894035587087274,1.8413150649517775,1.3098403922715807,400,0.1429921105504036,0.2337099239230156,0.3749369978904725,0.25055740708485247,400,7.626137923007488,9.600311791921143,11.952683088928085,9.705343091456633,400,0.476633620187968,0.6000194869950715,0.7470426930580053,0.6065839432160396,400,0.022574117660033616,0.04444084859165792,0.07320318598202107,0.04680265130552581,400,0.13587146788780108,0.18759420410807082,0.2631369756070376,0.19310309000227505,400,0.45005938469798207,0.5335019337487386,0.619274726205423,0.5353947680730584,400,0.7267400722189363,0.8004786169485665,0.8826859107315967,0.80516347697214,400,0.02166898020422878,0.05610149385743507,0.0909794361858572,0.055732782974919334,400,9.0,11.0,12.0,10.6975 +fresh-zh-2,2,3,Q,16,full_attention,200,0,200,0.8993129206413869,1.2311510134313721,1.6292167951818555,1.2540482087324198,200,0.2186799481511116,0.2791487127542496,0.3730020523071289,0.2890693274885416,200,6.687009320239848,8.75514697050005,10.684212423050207,8.69340081627616,200,0.4179380825149905,0.5471966856562531,0.6677632764406379,0.54333755101726,200,0.033169116600978064,0.055166760553339904,0.09284683362656626,0.06026045310170086,200,0.16976076946230767,0.2324586251796143,0.30716784535479696,0.23772766452125738,200,0.49194071151317953,0.554654697233007,0.6344001867663099,0.5589209980961543,200,0.7529814952689862,0.8171522256914825,0.8711913296012616,0.8150045580034063,200,0.01444950653251639,0.04395989360405843,0.07367107880632467,0.044640012674247345,200,9.0,11.0,12.0,10.52 +fresh-zh-2,2,3,K,16,full_attention,200,0,200,1.0167563085386064,1.289182523963973,1.5362084539839995,1.2801181593898219,200,0.21695202440023423,0.26905012130737305,0.3217982947826385,0.26668707966804506,200,7.563577773060466,9.27945438139734,10.974267067736548,9.17131253752412,200,0.4727236108162791,0.5799658988373337,0.6858916917335343,0.5732070335952575,200,0.030532988614936016,0.04828457990698644,0.07437327256193554,0.05230085879146733,200,0.1575994258889159,0.21977768003671322,0.2609350017693238,0.21351249858076077,200,0.4765393289006954,0.5474514630882964,0.6229359099881486,0.54906189132233,200,0.7526505803404225,0.8173710656563103,0.8757001322521665,0.818317140280337,200,0.013357270324334543,0.042232139039857286,0.06564683017398996,0.04157666276975659,200,9.0,10.0,12.0,10.4 +fresh-zh-2,2,3,V,16,full_attention,200,0,200,1.0811328606912867,1.4131184845464304,2.193098283978179,1.4891596455674152,200,0.24811245501041415,0.3043006360530853,0.4012023299932479,0.32522410213947295,200,7.371213190418436,9.117027987828546,10.661722116131328,9.003742174628126,200,0.46070082440115223,0.5698142492392841,0.666357632258208,0.5627338859142579,200,0.033379738211350735,0.05033171447340033,0.0780404188160741,0.05405035002244855,200,0.1771559038243438,0.219349370060362,0.27829204315840095,0.2235415539160946,200,0.5046271814643796,0.5473044951269688,0.6033912044456409,0.5514195652243391,200,0.7468725748336894,0.8078926787368341,0.8496249050365182,0.8023260215154692,200,0.016500361764635697,0.04968735000466541,0.07686336993421657,0.050650306058917,200,10.0,11.0,12.0,10.9 +fresh-zh-2,2,3,QKV,16,full_attention,200,0,200,1.0465959789813497,1.326384887623135,1.8111236160621047,1.3640863217425068,200,0.23445787727832795,0.28515270352363586,0.3634243905544281,0.2942305117100477,200,7.3752083316840364,9.263852056851638,10.973150670801747,9.116421928105702,200,0.4609505207302523,0.5789907535532274,0.6858219169251092,0.5697763705066063,200,0.030540504034247008,0.04847636544377003,0.07796202004584099,0.05271926403711071,200,0.16824336792511158,0.220655243402105,0.2776208168794974,0.2207363346845586,200,0.48328261869321454,0.5376425134375862,0.6122147155048353,0.5465372692511494,200,0.7475225662214877,0.8073752435269923,0.8596723189911747,0.8073101963329276,200,0.01504699354534652,0.04803234550828406,0.0742059194321406,0.04670363722708493,200,10.0,11.0,12.0,10.705 +fresh-zh-2,2,3,JS,16,full_attention,200,0,200,0.033448167066580936,0.05564773845230775,0.09787187126348726,0.06004451266739807,200,0.013152833096683025,0.027376949787139893,0.06018529832363128,0.03327927593141794,200,1.4727544829126265,3.3988920187533527,5.725476907183045,3.5449320107920017,200,0.09204715518203915,0.21243075117208454,0.3578423066989403,0.2215582506745001,200,0.11963532755144395,0.24716110088194726,0.6576526947945271,0.3167158478088197,200,0.31645763911178215,0.4970100259065895,0.8216160846783133,0.5330661500627615,200,0.6906147329649192,0.8098432117780925,0.9257492856690333,0.8062379211162343,200,0.8846961358686755,0.9365625382050167,0.9766278558195673,0.9333179735345538,200,0.003490389515842565,0.011699572151314348,0.023815301075335624,0.012853122026607502,200,3.0,7.0,9.0,6.6 +fresh-zh-2,2,3,sqrtJS,16,full_attention,200,0,200,0.5147837075870484,0.6829850850626826,0.8971190208103508,0.6926104764753837,200,0.11468573287129402,0.16545837372541428,0.24532595425844192,0.17407447088509798,200,5.304095096025002,9.12824132284305,10.916963993062089,8.593923252709432,200,0.33150594350156265,0.5705150826776906,0.6823102495663805,0.5371202032943395,200,0.031040645558570792,0.05018688622426508,0.134436893872321,0.06729200681748726,200,0.1707759904195831,0.233337792425879,0.3886315688481297,0.25794270892352494,200,0.4727869244530316,0.5464853478854644,0.6427440411339127,0.5502366009148489,200,0.7302245911601384,0.7823344771146692,0.8386868279110486,0.7832864792369361,200,0.04568047562806946,0.0678677686395084,0.08999645419244942,0.06769614052735536,200,10.0,11.0,12.0,11.305 +fresh-zh-2,3,4,Q,8,linear_attention,400,0,400,0.6812251007184387,1.347758723422885,2.8742594607174397,1.5544886476011015,400,0.14631469398736954,0.32208526134490967,1.03531209230423,0.4470309623144567,400,4.183795314399322,6.163438859857621,6.9655024985692675,5.941844610543826,400,0.5229744142999152,0.7704298574822026,0.8706878123211584,0.7427305763179782,400,0.0212167631324211,0.042568143690182185,0.13030612791783958,0.05706392803814369,400,0.19910666770396923,0.2476915829014677,0.362128983134063,0.2667700435484291,400,0.3783567651201555,0.44734498124279914,0.6428902775944623,0.4692515810034629,400,0.6580119006456923,0.7200698254221183,0.8319289083940477,0.7266867168399176,400,0.06156206019751065,0.10927652851642485,0.14428019309986914,0.10781034049159327,400,6.0,7.0,7.0,6.57 +fresh-zh-2,3,4,K,8,linear_attention,400,0,400,0.9337062157690525,1.3523602075874805,2.1561363413929944,1.4563357652071864,400,0.2000514656305313,0.3472091853618622,0.6602227389812471,0.3932704356685281,400,4.9200332157282824,6.321727312008696,6.999426341415524,6.1222504946924685,400,0.6150041519660353,0.790215914001087,0.8749282926769405,0.7652813118365586,400,0.02042297549666403,0.037925324284111855,0.08942959221074512,0.04828150176645309,400,0.20008027920921417,0.24797535728015935,0.35064883334782226,0.2636540782231443,400,0.3718783672438628,0.432800571066544,0.5533483189438939,0.4498190724506456,400,0.6508311553664654,0.7070849375802888,0.7724978524235098,0.7087244339703611,400,0.08577013610773004,0.11524661707701989,0.14426106552295564,0.11528385610996399,400,6.0,7.0,7.0,6.74 +fresh-zh-2,3,4,V,8,linear_attention,400,0,400,0.7077728314325213,1.2413044888526201,2.2631957776844502,1.381606738830451,400,0.16702189147472382,0.3244937062263489,0.6462681472301484,0.37337769774720075,400,4.7608679071955144,5.999111519972111,6.851865709534093,5.914010623821941,400,0.5951084883994393,0.7498889399965138,0.8564832136917616,0.7392513279777426,400,0.023937890009442325,0.04764740884695917,0.09719531029984338,0.05476326114677928,400,0.20566608030694966,0.25653797994259797,0.3520709877404495,0.267399722841454,400,0.3848867739509728,0.4569484863148974,0.56399790334758,0.4659834068370273,400,0.6632029797539796,0.727876926916892,0.7970127944977382,0.7298291915922949,400,0.07123923823826649,0.09915328469466353,0.13242126257114709,0.10052323652296803,400,6.0,6.0,7.0,6.4675 +fresh-zh-2,3,4,QKV,8,linear_attention,400,0,400,0.9467487499117851,1.4166230261325836,2.35920599475503,1.543510935259983,400,0.2062367469072342,0.3535090535879135,0.7230001866817475,0.41168989937752487,400,4.745075705508948,6.2622365822066515,6.958706838335711,6.078277407573788,400,0.5931344631886185,0.7827795727758314,0.8698383547919639,0.7597846759467235,400,0.0213770100707637,0.03964269212305083,0.09799736235695997,0.04943238097338261,400,0.19942511745931615,0.24496979921816864,0.341878624569254,0.25852253254751895,400,0.3742702647196632,0.437904650222272,0.5785693670545975,0.45644312845934715,400,0.6583957585201634,0.7094064337440728,0.7843585312849737,0.7171820403920584,400,0.08264559999431385,0.11060451779074318,0.1392998240635045,0.11111655747988841,400,6.0,7.0,7.0,6.7075 +fresh-zh-2,4,5,Q,16,linear_attention,400,0,400,0.6286894285003655,1.023394260671921,1.3528277426841668,1.0210299312572897,400,0.08047708421945572,0.1517123281955719,0.23211983442306522,0.1568526402208954,400,10.078752559390589,12.107719200494873,13.4157321480197,11.883582832278451,400,0.6299220349619118,0.7567324500309296,0.8384832592512312,0.7427239270174032,400,0.012841978012980766,0.021431439793240532,0.03916654498451242,0.02429751935594066,400,0.11543657623083156,0.14550026004524275,0.2002310471233691,0.15227665477897195,400,0.3934583656169914,0.444474335844636,0.516342190934689,0.45133342214798944,400,0.6709230779048313,0.7161289018834969,0.7721637466956828,0.7193877395961513,400,0.0625106112720948,0.09396193909089552,0.13038077800687706,0.09521750506500487,400,11.0,12.0,13.0,12.3325 +fresh-zh-2,4,5,K,16,linear_attention,400,0,400,0.6756591955432668,0.9881969420239329,1.3682786277262493,0.9902907327602225,400,0.08710963502526284,0.13617576658725739,0.22272974252700806,0.14834913161583244,400,10.505437210261272,12.31591344109961,13.77168252753122,12.181153891178965,400,0.6565898256413295,0.7697445900687256,0.8607301579707013,0.7613221181986853,400,0.010786953590961434,0.019942179443998377,0.034868058551460475,0.02213756472952461,400,0.11307066428167543,0.1396475419145041,0.192297017683191,0.14751539672280986,400,0.3856588134895072,0.43584386388507257,0.5037758442279161,0.44045477664972,400,0.6574902371969683,0.7114485202817505,0.7619465602107708,0.7098446059154567,400,0.06871823786379341,0.10285639334102706,0.13676094259995752,0.10263977095556345,400,12.0,13.0,13.0,12.535 +fresh-zh-2,4,5,V,16,linear_attention,400,0,400,0.33955027741612864,0.6936745375860482,1.3572318752645522,0.7828088770618706,400,0.04852813109755516,0.10321569442749023,0.23216074407100684,0.12352707331534475,400,9.895457863841646,11.645734236735755,13.094130106849045,11.532941988653661,400,0.6184661164901029,0.7278583897959847,0.8183831316780653,0.7208088742908538,400,0.01479477835230403,0.024926249377244282,0.04112715313767028,0.027368277465374843,400,0.11412034662942537,0.15081015220885047,0.20294101355765598,0.15689000298649813,400,0.40354943548239824,0.4620349303967301,0.5230834939119356,0.4649843192000263,400,0.6842118232162278,0.7311550601734638,0.7842384897525098,0.7329094030845161,400,0.059015179055331525,0.0860165543852225,0.12279736249502948,0.08854098099669828,400,11.0,12.0,13.0,12.1425 +fresh-zh-2,4,5,QKV,16,linear_attention,400,0,400,0.6803235459141433,0.9590906246448867,1.3362675739452243,0.9925166586854175,400,0.08297985419631004,0.13207364827394485,0.20362010896205907,0.13969412707723677,400,11.116112891934842,12.615717813416758,13.882878270204255,12.508566284410977,400,0.6947570557459276,0.7884823633385474,0.867679891887766,0.781785392775686,400,0.010166588456327817,0.017883951942293745,0.029290247985612437,0.019320086309333642,400,0.10860822300326076,0.134895417963523,0.17631712250943626,0.1397920380775314,400,0.3798330500245528,0.4225091923346047,0.48424898226319996,0.42810241109424096,400,0.65513195992357,0.6983106312296545,0.7483417680315073,0.6999279898335812,400,0.07414547408630082,0.10321727805164574,0.1389003629099275,0.10554997819592708,400,12.0,13.0,13.0,12.5775 +fresh-zh-2,5,6,Q,16,linear_attention,400,0,400,0.850007483549416,1.2207114743068814,1.6831544047221543,1.2485097377817147,400,0.11453252360224724,0.16855048388242722,0.2544349640607834,0.17848098322749137,400,10.872768914972001,12.457234886402812,13.636162416028718,12.342269318868563,400,0.6795480571857501,0.7785771804001758,0.8522601510017949,0.7713918324292852,400,0.01155672969773123,0.01895964406034801,0.03143775779058722,0.020533243078545666,400,0.11274584940351044,0.13842067691862875,0.1822133877041225,0.14285794531054014,400,0.3926102521609784,0.4365892174531998,0.49888747030841885,0.44202879903888587,400,0.6626627091061184,0.7072325138466298,0.7504335854905022,0.7062846815958508,400,0.0841811428762516,0.1103423577695062,0.13414125319742762,0.11048018698250914,400,12.0,13.0,13.0,12.7725 +fresh-zh-2,5,6,K,16,linear_attention,400,0,400,0.8992205899208784,1.2125543355941772,1.685433608107269,1.2559091555676423,400,0.10434456542134286,0.1562695950269699,0.250750732421875,0.1667019068636,400,11.699067397567521,13.274039164365778,14.172441952305217,13.078471225575745,400,0.7311917123479701,0.8296274477728611,0.8857776220190761,0.8174044515984841,400,0.008596768755171992,0.013690688909409839,0.024508766341189973,0.015421824734514056,400,0.10457791566060448,0.12681720619814252,0.16361390382919533,0.13141070948655018,400,0.36737523538138445,0.4080473915838079,0.4698767603393026,0.41403344744608866,400,0.6398465900608583,0.6741324020329855,0.7241800988974824,0.6773810139252291,400,0.10004589344021192,0.12898193555515566,0.15100716246065313,0.1270813694255518,400,12.900000000000006,13.0,14.0,13.1325 +fresh-zh-2,5,6,V,16,linear_attention,400,0,400,0.6542597630061209,1.077013324946165,1.7728758189827205,1.1508489729938447,400,0.09124561175704003,0.1528458371758461,0.2834826081991196,0.1728745711222291,400,10.395379565008483,12.15055368523931,13.61100414640209,12.088915622102425,400,0.6497112228130302,0.7594096053274568,0.8506877591501306,0.7555572263814015,400,0.011701296434203699,0.02112091481908649,0.03594324393468738,0.022964505938502328,400,0.113934423543713,0.14358232820954148,0.19249449654720718,0.14994396763834159,400,0.38844467061771454,0.4443463220973173,0.5124238947536248,0.44704203676657445,400,0.6611759888894667,0.7084386049916309,0.7608543647408813,0.7101876785265088,400,0.07741190305724786,0.10679659189136748,0.13734503220693448,0.10662731972143934,400,12.0,13.0,13.0,12.6275 +fresh-zh-2,5,6,QKV,16,linear_attention,400,0,400,0.9629941495135427,1.2636645091697574,1.6333377357572318,1.2833677584212273,400,0.11412648484110832,0.1560099869966507,0.2304790332913399,0.16487920071929693,400,11.995346718416863,13.335323521264634,14.1945670553286,13.202256890120688,400,0.749709169901054,0.8334577200790396,0.8871604409580375,0.825141055632543,400,0.008479455338460208,0.013321398183935436,0.022256740302641128,0.014492246329915956,400,0.10545687966634136,0.12368967730773128,0.15644732659478286,0.12798491264495473,400,0.3688232426808109,0.40520846065543653,0.45807652748344146,0.41019189419911184,400,0.6396676037041121,0.6730875467277726,0.7106469009746262,0.6751986841323167,400,0.10350578991329562,0.12812625766121605,0.15125214859158023,0.12782037100290775,400,13.0,13.0,14.0,13.1375 +fresh-zh-2,6,7,Q,16,full_attention,200,0,200,0.687014796352014,0.9055734653957188,1.2147770636714994,0.932920820217114,200,0.16238171160221102,0.19910967350006104,0.2476297616958618,0.2027275863289833,200,7.008896455272861,8.992261441986745,11.137891337707893,8.97713126591093,200,0.4380560284545538,0.5620163401241716,0.6961182086067433,0.5610707041194332,200,0.029102636838313784,0.05195500660458982,0.0855209851440279,0.056306682563575045,200,0.16191361667546386,0.21855865222851922,0.29444711878290125,0.2244740129273826,200,0.48331582232369974,0.5603641535063217,0.648928652276827,0.5639449492990658,200,0.7451328104195889,0.8033425773136801,0.8596025650444639,0.802019949625251,200,0.03756738902111045,0.05906582746625982,0.08772425071656301,0.06087593837498951,200,10.0,11.0,12.0,10.98 +fresh-zh-2,6,7,K,16,full_attention,200,0,200,0.7348533940268681,0.9835070561384782,1.1769101968966422,0.9729980131541379,200,0.14996416419744493,0.1923513114452362,0.2304789423942566,0.19173243373632431,200,7.887335732306829,9.620315343486261,11.223242560317127,9.552603305697515,200,0.4929584832691768,0.6012697089678913,0.7014526600198204,0.5970377066060947,200,0.028378696523159885,0.04420981366549716,0.06865807014485487,0.04735011317381043,200,0.16371093420458147,0.19230274555962595,0.2520936745368967,0.20029772434589,200,0.4656664448419066,0.5388826987821724,0.6348736730542034,0.5474517733604969,200,0.7417436587453164,0.7907441237863757,0.8427634295916976,0.793464417069535,200,0.04117053136310003,0.060855559900653736,0.08807468500585407,0.06453110082590861,200,10.0,11.0,12.0,11.16 +fresh-zh-2,6,7,V,16,full_attention,200,0,200,0.9429454141529278,1.2080861646682024,1.4355560445226727,1.1927375790756196,200,0.17252458184957503,0.21075499802827835,0.2731092512607574,0.2185354572534561,200,7.8281428435786475,9.947087921110228,11.859746917108396,10.02382429166331,200,0.48925892772366547,0.6216929950693892,0.7412341823192747,0.6264890182289569,200,0.02327372004345265,0.04056756724316039,0.06959407544049823,0.042228891303031556,200,0.14104365117909387,0.17937449700728103,0.23899880264527093,0.18723263655185207,200,0.4517469947525151,0.5302476337416329,0.6134079113137781,0.5286569063336067,200,0.7270124096591025,0.7830081701716999,0.8331478690273625,0.7828620819653583,200,0.0457122390610176,0.06947608514683806,0.09520029416258481,0.0702036914792069,200,10.0,12.0,12.0,11.48 +fresh-zh-2,6,7,QKV,16,full_attention,200,0,200,0.8218812965322286,1.0770739214494824,1.2469542897306383,1.0517673873191233,200,0.1626240238547325,0.2037961408495903,0.24288544803857803,0.20300193943083286,200,7.655344856465389,9.707328259401525,11.612704653324975,9.730204981630752,200,0.4784590535290868,0.6067080162125953,0.725794040832811,0.608137811351922,200,0.025186759288658715,0.04321621995332453,0.07267003375080719,0.04537094552500097,200,0.14522274418396963,0.19556793573579595,0.2502941221608022,0.1967809943997404,200,0.4590558288460326,0.5390097153036804,0.6216716665349686,0.5399644871645356,200,0.7293553921357455,0.786360543772143,0.8406465958743105,0.7882964969244451,200,0.04172663492609001,0.06605676506088651,0.09306152261354463,0.06697438966416348,200,10.0,11.0,12.0,11.3 +fresh-zh-2,6,7,JS,16,full_attention,200,0,200,0.015560420188137416,0.03576696577965777,0.06376001235039438,0.03857223819759504,200,0.004956961330026388,0.012278192676603794,0.027893566712737083,0.014450385831296444,200,2.3441436284271195,4.85312814600349,7.294545866668523,4.932494843897084,200,0.14650897677669497,0.30332050912521813,0.4559091166667827,0.30828092774356775,200,0.07956148857028608,0.1531252197288483,0.38839506587827133,0.19481818163872655,200,0.20943451843543534,0.33797916279509155,0.6335231511834918,0.3776489454393901,200,0.6533904097570001,0.7909383258385765,0.904002350934321,0.7811691843380018,200,0.8894584007339179,0.9456311054989818,0.9804330583997438,0.9389277564448875,200,0.003463397057282541,0.009868600105001825,0.02267380658795696,0.011757646771249202,200,4.0,7.0,9.0,6.665 +fresh-zh-2,6,7,sqrtJS,16,full_attention,200,0,200,0.3461533321416937,0.5831202674889937,0.7570074148476124,0.5766709119178995,200,0.07040502727031708,0.11080574616789818,0.16701333373785018,0.11430789001286029,200,6.8480609902259015,9.477582448189732,11.462535995996191,9.2879402417683,200,0.42800381188911885,0.5923489030118583,0.716408499749762,0.5804962651105188,200,0.0263901150613913,0.04587963140386321,0.08909532664702488,0.05404535160656519,200,0.13910662840989046,0.18719585664591068,0.30449170860394,0.20526591532690439,200,0.4654513551144829,0.550799964667037,0.6596268322314756,0.5568742351932628,200,0.7499024924051205,0.8036992126753697,0.8627447404576886,0.8025792324184949,200,0.03819984821398796,0.061363312641139395,0.08371504206415634,0.06124599579158597,200,10.0,11.0,12.0,10.98 +fresh-zh-2,7,8,Q,8,linear_attention,400,0,400,0.963251867890358,1.558474475517869,2.902129247784615,1.768479774366133,400,0.21639974415302277,0.34893959760665894,0.682628345489502,0.39776828300207856,400,5.900423334843829,6.562370328329661,7.080570394786534,6.506091108670474,400,0.7375529168554786,0.8202962910412076,0.8850712993483167,0.8132613885838093,400,0.018550353918216182,0.03129597278717164,0.0508335610929049,0.03386220270966255,400,0.19099125089792265,0.2196247004974251,0.2713292937130322,0.22684340191967664,400,0.3622199646459188,0.40477335744870974,0.4642547751042748,0.4096773772989637,400,0.6460621746132328,0.7016725450935115,0.7571179779152897,0.701532640152621,400,0.0898226019973526,0.11605280310423503,0.14095359940070457,0.11532923666423305,400,6.0,7.0,7.0,6.7625 +fresh-zh-2,7,8,K,8,linear_attention,400,0,400,0.3344550456851721,1.423342164605856,2.4089005753397945,1.4791520859807497,400,0.06650525331497194,0.3037361800670624,0.5272434651851655,0.31955348477698864,400,6.140099928493848,6.648216936994127,7.195414514235623,6.654039093674819,400,0.767512491061731,0.8310271171242659,0.8994268142794529,0.8317548867093524,400,0.015974179899856118,0.029047188716045138,0.04327291558665942,0.02977082736489778,400,0.18255935248950678,0.21223812513102963,0.25374748763545574,0.21581782644832345,400,0.34909672992970164,0.39511687923265365,0.4417615451872795,0.3966416971136322,400,0.6408602874835513,0.6959855803306833,0.7469270760870556,0.6940795298593521,400,0.0940784079775587,0.11884592939527396,0.147895087150434,0.12030877218272039,400,6.0,7.0,7.0,6.8325 +fresh-zh-2,7,8,V,8,linear_attention,400,0,400,0.6318522274494172,1.3380410242825747,2.1230420328676702,1.345673517389223,400,0.15563115626573562,0.32925964891910553,0.6325971961021424,0.35993642058223485,400,4.903432227290967,6.160560485729121,6.91263691622773,6.00146375175668,400,0.6129290284113709,0.7700700607161401,0.8640796145284663,0.750182968969585,400,0.02247155698121308,0.04265493453018222,0.0902159408857469,0.051546919732455176,400,0.2022139748083212,0.244284156422943,0.34055198039223844,0.26259898075292787,400,0.3814515974538133,0.43535256343513834,0.5395020133405021,0.44914336529379584,400,0.6641321417914857,0.7318753402200763,0.8047708129765924,0.7335524132015949,400,0.06933501272639045,0.09939702144086002,0.135852868034634,0.10095495073171268,400,6.0,6.0,7.0,6.465 +fresh-zh-2,7,8,QKV,8,linear_attention,400,0,400,0.9557942457497121,1.5293590724468231,2.41666070073843,1.626029323064722,400,0.20811364352703096,0.33834849298000336,0.5384937226772308,0.3625067052990198,400,5.907443050322293,6.559081406945095,7.102326096958445,6.523263399984855,400,0.7384303812902866,0.8198851758681369,0.8877907621198057,0.8154079249981069,400,0.01805593884275101,0.031383289997399935,0.05060340784815262,0.03337498319641826,400,0.1884747477101536,0.21846092429215216,0.27018820587481346,0.22556789452628564,400,0.36405292852977306,0.4033010275168095,0.4585141255907992,0.40808034620775574,400,0.6482858229221864,0.704909689497271,0.7565942537443274,0.7036711339352562,400,0.09312962317383842,0.11528445176589353,0.14417956065956603,0.11736644574938487,400,6.0,7.0,7.0,6.77 +fresh-zh-2,8,9,Q,16,linear_attention,400,0,400,0.6823213030584157,1.0089960535988212,1.4070010952651502,1.031631159793469,400,0.08882656767964364,0.137319877743721,0.21284338980913167,0.14597560971044005,400,10.748178703766573,12.511213241360938,13.641956624219524,12.340986035990461,400,0.6717611689854108,0.7819508275850586,0.8526222890137203,0.7713116272494038,400,0.01152348591295907,0.018590211064092893,0.03257495390578433,0.020549881591731177,400,0.10759858741312078,0.13622070618983229,0.18389406344123801,0.14179697323663387,400,0.3821298673679192,0.43214380762449245,0.48966568631700197,0.4351944699740531,400,0.6651459708125766,0.7110202289609098,0.7525544246429529,0.7101317043045845,400,0.07796729177633167,0.1022986429519939,0.1284634274749423,0.10307941565792937,400,12.0,13.0,13.0,12.56 +fresh-zh-2,8,9,K,16,linear_attention,400,0,400,0.6854124701581895,0.9712583301588893,1.3763784609735012,1.0048457650234923,400,0.08375069126486778,0.12543070316314697,0.18150667846202853,0.13143930226564407,400,11.593289276001764,12.877356521022499,13.876507198131888,12.774017038994762,400,0.7245805797501103,0.8048347825639062,0.867281699883243,0.7983760649371726,400,0.010201860353355522,0.01616628120203887,0.025340630279753015,0.017265634813648602,400,0.1061466974181515,0.1277630621977717,0.1608421893066332,0.13139535149490755,400,0.37893102713145815,0.4174831238683251,0.4661283096109969,0.42162050177387655,400,0.6576761503961365,0.6971205383448346,0.7397410067470522,0.6974380469006896,400,0.08452968579316675,0.10877649617766658,0.13643348469889496,0.10955996231511843,400,12.0,13.0,13.0,12.73 +fresh-zh-2,8,9,V,16,linear_attention,400,0,400,0.48543434822931886,0.8241976434364915,1.390811233967543,0.8942763623344945,400,0.0663076288998127,0.11690865457057953,0.2089322805404663,0.1287225678190589,400,10.79117437602816,12.417040104932138,13.56294164054711,12.239513330698054,400,0.67444839850176,0.7760650065582586,0.8476838525341944,0.7649695831686284,400,0.011979009652157922,0.01923679148830871,0.03217954245640139,0.021378330360068025,400,0.11057222294310148,0.13727640665752766,0.17959854698790725,0.14342703436737728,400,0.39087907857799525,0.4327445721995613,0.49177174137234836,0.43870757532896176,400,0.6693246786995476,0.7107624183831687,0.7574260801290429,0.7133601426091758,400,0.07671675774592389,0.09978468672973495,0.1278380518592144,0.10083642270736157,400,12.0,12.0,13.0,12.48 +fresh-zh-2,8,9,QKV,16,linear_attention,400,0,400,0.7290561510249972,1.0042466334998608,1.3689184928312899,1.0303375706344378,400,0.08791246861219407,0.12238382175564766,0.1793705210089684,0.12919468224048614,400,11.891351255688503,13.15485743654848,14.114522585908896,13.08378388435116,400,0.7432094534805315,0.82217858978428,0.882157661619306,0.8177364927719475,400,0.00890564443716252,0.014418717867100896,0.023034381130956304,0.015257512185827396,400,0.1017899963904398,0.12088552065530515,0.1543943061055882,0.12573250635990207,400,0.3689747623263646,0.4059591609612434,0.45284355329091996,0.4082877155416354,400,0.645278838906934,0.686243759390091,0.7287286192290482,0.6864826503266932,400,0.08656954896627078,0.11623962525946577,0.14225465494600753,0.11512100345003713,400,12.0,13.0,13.0,12.8325 +fresh-zh-2,9,10,Q,16,linear_attention,400,0,400,0.5940493597649038,0.9190406142733991,1.4654270390048625,0.991110304776812,400,0.07924356609582901,0.139134019613266,0.2585415571928024,0.16127049322240056,400,9.753563666662687,11.986198648377673,13.38104664082704,11.70454171084571,400,0.609597729166418,0.7491374155236046,0.83631541505169,0.7315338569278569,400,0.013048081307364522,0.02232457238713536,0.042695197127111145,0.026889678524774082,400,0.11618364535892516,0.1485430757572613,0.2130188196134589,0.15993091659907313,400,0.39648567697139103,0.45344891947225324,0.5284251642445584,0.45926066604330096,400,0.6743541627891385,0.721489767013564,0.7724609728037911,0.7236436618935378,400,0.07222800779220662,0.09724282856028803,0.12691919531479426,0.09786621235768816,400,12.0,12.0,13.0,12.375 +fresh-zh-2,9,10,K,16,linear_attention,400,0,400,0.6508710502646864,0.8990921657532454,1.2014237709343434,0.9115452453627949,400,0.09279014691710472,0.12802980840206146,0.19691963344812394,0.1370885619148612,400,10.74453195327038,12.34017603015284,13.602691813339584,12.189079767266808,400,0.6715332470793988,0.7712610018845525,0.850168238333724,0.7618174854541755,400,0.011749191769017274,0.019771864115134357,0.032608636491846615,0.021777674339640524,400,0.11394347840216071,0.14518094824920502,0.20033669338962018,0.15084674261926462,400,0.3895745353366957,0.43567503479854836,0.4905992150419507,0.43957418600189624,400,0.6648104265002206,0.7060110986010514,0.7495502527362722,0.7060526669637863,400,0.08216459319774563,0.10578297165027356,0.12957213651090327,0.1060847691369965,400,12.0,13.0,13.0,12.6325 +fresh-zh-2,9,10,V,16,linear_attention,400,0,400,0.4587136392015964,0.6876073661260307,1.1429073590785266,0.7556872137007303,400,0.0620508462190628,0.0999692864716053,0.18130848407745362,0.1131087754573673,400,10.541399798671643,12.303897584476786,13.767460085430711,12.138124354747667,400,0.6588374874169777,0.7689935990297991,0.8604662553394195,0.7586327721717292,400,0.010810720346346954,0.020026732140634166,0.03452335830916909,0.022868920537781206,400,0.11040454622069198,0.13996767442632246,0.19726316057855317,0.1503242189004077,400,0.38354830855961913,0.4377839536513496,0.5014292798009696,0.44172213673418825,400,0.6611077640452623,0.713401778786793,0.7597113155373733,0.711754891063228,400,0.07511016115775464,0.10294252807591586,0.12925376727421692,0.10352169101921159,400,12.0,13.0,13.0,12.5575 +fresh-zh-2,9,10,QKV,16,linear_attention,400,0,400,0.6504405225161464,0.9181888550519943,1.2623574163764715,0.9357619134231936,400,0.08352749869227409,0.12333659827709198,0.19588928669691086,0.13284218261018396,400,11.081968736924063,12.707509175288696,13.87518271982123,12.559380002267336,400,0.6926230460577539,0.7942193234555435,0.8671989199888269,0.7849612501417085,400,0.01020919803222796,0.017273204857219733,0.029585796039963478,0.01912899162313016,400,0.10936519121613489,0.1344591727060116,0.181513514488691,0.14130480498850434,400,0.38238546355804903,0.4246176193728921,0.4811406762594599,0.4285852988874659,400,0.6541239591633542,0.6986099392224142,0.7409833917545308,0.6983710013500135,400,0.08656914839996535,0.11016001537868028,0.13871677250801,0.11116320279053067,400,12.0,13.0,13.0,12.745 +fresh-zh-2,10,11,Q,16,full_attention,200,0,200,0.6182043068110943,0.8430793648585677,1.0857965022325515,0.8497080951323732,200,0.0900506965816021,0.11527549847960472,0.16624964475631712,0.12279644653201104,200,10.296525789731087,12.356836881317625,13.865081951346102,12.289314249642585,200,0.6435328618581929,0.7723023050823515,0.8665676219591314,0.7680821406026616,200,0.010265282438653328,0.019655317920679517,0.03692827119913988,0.02112511709810059,200,0.11166262597596695,0.14000373394677745,0.19496539860242063,0.14664508779155078,200,0.3856706382286854,0.43715747422651474,0.5028382225343072,0.44287381136587356,200,0.6551924879627492,0.7048965002033045,0.7593125302658268,0.7058761713593004,200,0.08730725467807957,0.11136438817902772,0.13760904003980043,0.11237746116746267,200,12.0,13.0,13.0,12.735 +fresh-zh-2,10,11,K,16,full_attention,200,0,200,0.6179400110151619,0.8210435467772186,0.9326142269186675,0.8059216461423785,200,0.09643219113349914,0.19087949395179749,0.2463380768895149,0.1806161604821682,200,6.395476940723292,9.99831375902677,12.420090155643402,9.656327367275795,200,0.39971730879520573,0.6248946099391731,0.7762556347277126,0.6035204604547372,200,0.019224453262921343,0.04002139336382005,0.10013431684833303,0.050616146198025104,200,0.13967444699967446,0.21820106843747633,0.3313994440422073,0.2241602844078995,200,0.44244810435715004,0.5233843591379896,0.6195085798000509,0.5269320214965911,200,0.6859930345749605,0.7623535927525094,0.8175421883435254,0.7608493605031171,200,0.06119630602474754,0.08536667613959595,0.12771267831618296,0.08911247711584032,200,11.0,12.0,13.0,12.04 +fresh-zh-2,10,11,V,16,full_attention,200,0,200,0.7490824665408582,0.9892853982746601,1.1872108437120914,0.9612219040002674,200,0.10072839632630348,0.12911496311426163,0.17608837038278569,0.13738574862480163,200,10.27789454315233,12.625539630481471,14.250811780267297,12.486220387327233,200,0.6423684089470206,0.789096226905092,0.890675736266706,0.780388774207952,200,0.0081828828793736,0.017819217818049907,0.03711771363240198,0.02003978766532885,200,0.10012299985045268,0.1413370047790768,0.17886571726664244,0.14389130370881914,200,0.3606295069968762,0.4203432078353291,0.511565509618861,0.4296444567132853,200,0.6479928108530383,0.7007020430941059,0.7523096721400566,0.6992044379475725,200,0.08438380254303705,0.10930967986622325,0.14483154075114857,0.11441816272357419,200,12.0,13.0,14.0,12.86 +fresh-zh-2,10,11,QKV,16,full_attention,200,0,200,0.6919123398605734,0.9175626114010811,1.079559326544404,0.8972398960427381,200,0.0953451782464981,0.14140773564577103,0.17902919054031372,0.14258599277585746,200,9.51595316669769,11.85988651912448,13.944805415968524,11.769389525385082,200,0.5947470729186056,0.74124290744528,0.8715503384980328,0.7355868453365676,200,0.009825522948296406,0.023272394708995434,0.04542622555126366,0.02562800807654317,200,0.11031296338462056,0.15235756008040896,0.226956520186547,0.16121381079474326,200,0.3895726605760364,0.45783505177297035,0.5417679219453154,0.4573380716566514,200,0.6557039264870891,0.7174907094368088,0.7712403004479108,0.7180762613104094,200,0.08018298408239406,0.10563455649338205,0.14052965922963975,0.10714902018395173,200,12.0,13.0,13.0,12.645 +fresh-zh-2,10,11,JS,16,full_attention,200,0,200,0.011142155992820335,0.024161406952771358,0.05232254761458533,0.032801581233288746,200,0.003320680861361325,0.007225634763017297,0.02942522913217544,0.01661940539430361,200,2.0824668656602046,5.565923289458514,8.888684606686917,5.610989218469195,200,0.13015417910376278,0.3478702055911571,0.5555427879179323,0.35068682615432467,200,0.05333640346721704,0.12498647316508793,0.4455497111445807,0.1908647128637297,200,0.20188493853436654,0.32884190496612775,0.6768494902535758,0.38083407850613216,200,0.5701547550098055,0.7081905078252608,0.8814072250084485,0.720534753865286,200,0.8270653845106197,0.892816767226302,0.9605770554679033,0.8942457305006778,200,0.008964640819787316,0.028237855713225096,0.051504934569925824,0.029574690313179733,200,5.0,9.0,10.0,8.125 +fresh-zh-2,10,11,sqrtJS,16,full_attention,200,0,200,0.3440644766204059,0.5116504959296435,0.702592519018799,0.5206322756764712,200,0.057624995708465576,0.08500373363494873,0.17153594493865965,0.10536864038556815,200,7.678969183860344,11.249454889883909,12.929734635907007,10.561917864766247,200,0.4799355739912715,0.7030909306177443,0.8081084147441879,0.6601198665478905,200,0.015830636730338177,0.028152752798739116,0.07237079286306858,0.04651089632551939,200,0.12516382305205778,0.1733862229785058,0.29591365308423623,0.19928878972405076,200,0.4119472567522575,0.479859237667988,0.609922620484741,0.49941370888429515,200,0.6884653752354623,0.7337170812536673,0.806508031618447,0.743431284538109,200,0.06520874060840567,0.09888416782934102,0.12499918941467338,0.09649088118711639,200,11.0,12.0,13.0,12.26 +fresh-zh-2,11,12,Q,8,linear_attention,400,0,400,0.7174284402281046,1.0416490640491247,1.6478107027709485,1.1249526698328554,400,0.17227262109518052,0.27689214050769806,0.43004508018493653,0.2909380738995969,400,5.645621815275392,6.449926361869339,7.081045764542591,6.3950593222030445,400,0.705702726909424,0.8062407952336674,0.8851307205678238,0.7993824152753806,400,0.018539604804619934,0.03433205988684744,0.05957535883532161,0.037465116468104596,400,0.20954179962414277,0.25607916430871275,0.3162627244999324,0.26137338519311787,400,0.3681165974330923,0.4202829622993868,0.48368482937354046,0.42253120718389603,400,0.6296631177812888,0.6771907016777992,0.7221899386211221,0.6768252797300872,400,0.10119994197081049,0.12883133220282608,0.15892517191080624,0.12913024163230447,400,7.0,7.0,7.0,6.91 +fresh-zh-2,11,12,K,8,linear_attention,400,0,400,0.49749523624777797,0.8568938840180635,1.4117163300514224,0.9094282926851883,400,0.11540010571479797,0.20011023432016373,0.3489511519670487,0.21609645273536443,400,6.043822515396557,6.659691705846258,7.162028930421003,6.621273616962107,400,0.7554778144245696,0.8324614632307823,0.8952536163026253,0.8276592021202633,400,0.016714559001961008,0.028750999085766917,0.04623794471768899,0.030501910135668765,400,0.19622450418611484,0.2362882154531355,0.28633817947605517,0.23966859516802466,400,0.36052695988212297,0.402609372860689,0.4568736220640487,0.4066498018362549,400,0.631065015164643,0.6722803808079466,0.7075053507232126,0.6709150642442728,400,0.09945442003098852,0.12928909497119184,0.1583392140054965,0.12956230568957544,400,6.0,7.0,7.0,6.8925 +fresh-zh-2,11,12,V,8,linear_attention,400,0,400,0.4062795072793961,0.7250700024887919,1.2533206272870305,0.7944815246178769,400,0.08800399526953698,0.1751338243484497,0.34314029514789585,0.2023499442823231,400,5.560829936103033,6.534286012569197,7.312159895626337,6.474463807889226,400,0.6951037420128792,0.8167857515711496,0.9140199869532921,0.8093079759861532,400,0.013438285259616456,0.03204447142635929,0.06266360213087237,0.03558724800804042,400,0.19200364281979748,0.24485629673412312,0.32306643322221906,0.250047916450663,400,0.3534570192916535,0.41497908154087076,0.48734722106830203,0.41812027094957654,400,0.619070358765415,0.6768373505910974,0.7363101659596577,0.6773377562745833,400,0.09251150706101313,0.1282973289698882,0.16825673005136343,0.1289814285863814,400,6.0,7.0,7.0,6.825 +fresh-zh-2,11,12,QKV,8,linear_attention,400,0,400,0.6409212961792946,0.9385962951928377,1.391756034642458,0.9910722961556166,400,0.14541933834552764,0.2368302196264267,0.3563231647014618,0.24466401603072882,400,6.004525983549825,6.6566621214750725,7.192973786348265,6.6209618926149565,400,0.7505657479437281,0.8320827651843841,0.8991217232935331,0.8276202365768696,400,0.016028066812898065,0.028829088196442303,0.04747548231989689,0.03058199008525468,400,0.20350660364092346,0.2448744209666256,0.2918471482228204,0.2470682383466615,400,0.36413013360327506,0.4029667037814628,0.45444828835227585,0.40555786196274096,400,0.6220582413985967,0.6659411751820361,0.7064438399770928,0.6652486300829922,400,0.10525267936265847,0.1334787753736142,0.16317027406969137,0.133907188587801,400,7.0,7.0,7.0,6.935 +fresh-zh-2,12,13,Q,16,linear_attention,400,0,400,0.4648180345073342,0.7350729533936828,1.1088959102518858,0.7650471870714682,400,0.06437505260109902,0.11736167967319489,0.25465229451656346,0.1422804689593613,400,7.980607377033428,11.69533179776774,13.321869667541915,11.138351863842743,400,0.4987879610645892,0.7309582373604837,0.8326168542213697,0.6961469914901715,400,0.013402279730941831,0.024537814695054873,0.06699203686475938,0.03406569641233204,400,0.11447574139603847,0.15820473462153312,0.27973523586790683,0.17874642243132513,400,0.392272832364534,0.45739460679120136,0.5735496447742738,0.47132516990443224,400,0.6741650432014945,0.7229972387716781,0.7920757180625667,0.7272785659305004,400,0.06116814672095206,0.09315098707856558,0.11986461303196094,0.09183475945491533,400,11.0,12.0,13.0,12.2075 +fresh-zh-2,12,13,K,16,linear_attention,400,0,400,0.43644047863781454,0.6150440508499742,0.885837840847671,0.640159845474991,400,0.05348301939666271,0.08606762066483498,0.1327272593975067,0.0909001738904044,400,10.609696154103341,12.677616228772958,13.779588089416505,12.451936450178504,400,0.6631060096314588,0.7923510142983099,0.8612242555885316,0.7782460281361565,400,0.01074251880213756,0.01747113164489639,0.033870302586026545,0.019924547551512118,400,0.10672404038918985,0.13686365831458364,0.18391520984723583,0.14209401828396476,400,0.37475586358785146,0.4261177217387738,0.4922604500991559,0.4284380684141332,400,0.6509313499019318,0.692884525846071,0.7518473928618353,0.6974306846052744,400,0.0717460074820675,0.10794732069578737,0.12984648555232933,0.104066446898042,400,12.0,13.0,13.0,12.565 +fresh-zh-2,12,13,V,16,linear_attention,400,0,400,0.36192337833344934,0.5174503484740853,0.7629639335907996,0.5474714133900125,400,0.04824961759150028,0.07939847931265831,0.13315983563661576,0.08685986062511802,400,9.416840673020513,11.963123635883033,13.475963072157349,11.701761853101697,400,0.588552542063782,0.7476952272426896,0.8422476920098343,0.7313601158188561,400,0.012486612733137557,0.022496222617642822,0.04660576524349302,0.0265630736741483,400,0.11357027502476336,0.14645015983607937,0.21395648293594152,0.15786361487844705,400,0.3901281183551474,0.4464846042842139,0.5343827986199315,0.45641917160208023,400,0.667675179684965,0.7184796332470252,0.7843973217928247,0.7228146271539679,400,0.06160708989755437,0.09363440932785222,0.1244287677160627,0.0930533610389719,400,11.0,12.0,13.0,12.2225 +fresh-zh-2,12,13,QKV,16,linear_attention,400,0,400,0.5083688435144722,0.6778793437406421,0.8920387076213957,0.6910350280703278,400,0.06358577683568001,0.09565698355436325,0.17257534116506582,0.10821326026692986,400,9.546250098573243,12.668280726527051,13.866791143715597,12.116743592150112,400,0.5966406311608277,0.7917675454079407,0.8666744464822248,0.757296474509382,400,0.010255757542072967,0.017533130541037473,0.0450700642902495,0.023622794630316815,400,0.10727398089184224,0.13726020547040674,0.23141649962308064,0.1546448287677016,400,0.3705335079787574,0.42385824880839074,0.5177988070129703,0.43508470169750696,400,0.6465405269444289,0.694501294792569,0.7639983296374885,0.7000638327140604,400,0.06860611407223591,0.10774452428590514,0.13226573060963456,0.10396430090013592,400,12.0,13.0,13.0,12.555 +fresh-zh-2,13,14,Q,16,linear_attention,400,0,400,0.4082620666828007,0.6566769070923328,1.018225465901196,0.6778513106654281,400,0.060181866958737375,0.11023851484060287,0.19894797205925002,0.12176891996059566,400,8.465786961617345,11.010122526973724,12.841313146668348,10.829122823069454,400,0.5291116851010841,0.6881326579358578,0.8025820716667718,0.6768201764418409,400,0.016398566466760522,0.03021388275218973,0.059333211474918546,0.034832449791367355,400,0.12627338408154037,0.16737049937173804,0.24848779552751718,0.17725217971336002,400,0.42015753979296167,0.48840600325812433,0.5763641912598081,0.49370033299612603,400,0.6906021369610973,0.7502718674030113,0.8098964045049183,0.7499606583242077,400,0.0612625858578231,0.08823254408201074,0.1209825393944216,0.089736052274174,400,11.0,12.0,13.0,12.0525 +fresh-zh-2,13,14,K,16,linear_attention,400,0,400,0.40258807097561655,0.6888491520658135,1.04599779099226,0.7074872706833412,400,0.06099251806735993,0.11867357045412064,0.21612842828035359,0.13288111188448964,400,8.550091775949394,10.603605615750073,13.054449071229888,10.692637264481775,400,0.5343807359968371,0.6627253509843796,0.815903066951868,0.6682898290301109,400,0.015042387981758307,0.03392805593006953,0.05808833287790811,0.036103055147632034,400,0.12579778146789344,0.1826946560538557,0.25023844797455763,0.18546139728183023,400,0.4158988941830746,0.49949255961975103,0.5748123811750302,0.4985183844562073,400,0.6796276595523479,0.7463054550056908,0.806235716460511,0.746261222118471,400,0.06230411210279405,0.09264426374823544,0.13081666588860644,0.09397789079780448,400,11.0,12.0,13.0,12.195 +fresh-zh-2,13,14,V,16,linear_attention,400,0,400,0.3138166696764529,0.5111526020336896,0.9712537154555322,0.5927337607345544,400,0.04693476036190987,0.09419038519263268,0.21543557941913608,0.11742577834054828,400,7.690330185508457,10.698868383418674,13.027735809722708,10.523291425166436,400,0.48064563659427856,0.6686792739636671,0.8142334881076693,0.6577057140729022,400,0.015209940458594914,0.0330323622138592,0.07203578349589589,0.039829669314783656,400,0.1250311956787199,0.18004793163259486,0.288645541884859,0.19300949855023955,400,0.41326130327374505,0.4972096256826726,0.5896553816791079,0.49829991715721805,400,0.6811902030439273,0.7530399083664919,0.8137324247319694,0.7481367493999028,400,0.06007598713202279,0.08774736808701211,0.12451700490227277,0.09020101363237004,400,11.0,12.0,13.0,12.0925 +fresh-zh-2,13,14,QKV,16,linear_attention,400,0,400,0.4332016982138157,0.672470421064645,0.9907066495157779,0.6982732560791192,400,0.06542271301150324,0.11074889078736305,0.20875086784362792,0.12477353234775364,400,8.730181880390525,11.161445519311744,13.194518068324355,11.058342691881542,400,0.5456363675244078,0.697590344956984,0.8246573792702722,0.6911464182425964,400,0.0141749930928718,0.02890040528809597,0.055514833536948416,0.03248472471492419,400,0.12174845187124765,0.16877728175672818,0.23896452146384592,0.17640131659257577,400,0.41250834581093465,0.4807066693548065,0.5599759311850458,0.4845135485442798,400,0.6769636787814528,0.7403742557581983,0.7973600027543113,0.738784916000768,400,0.06489034692144789,0.09614467525649267,0.13260979497935801,0.09773683295902462,400,11.0,12.0,13.0,12.3525 +fresh-zh-2,14,15,Q,16,full_attention,200,0,200,0.7097749772481621,0.947596563026309,1.1964474111795425,0.9440132132906001,200,0.1114270530641079,0.1416826918721199,0.17911817282438278,0.14351837493479253,200,10.161813035420382,11.605214187888468,12.747865354967766,11.531399366258693,200,0.6351133147137739,0.7253258867430292,0.7967415846854854,0.7207124603911683,200,0.017007482555948486,0.025246054184940317,0.038301624523822866,0.026543792843746708,200,0.12881754102864462,0.15163818040351373,0.18279252284558042,0.1540963114084707,200,0.4297963230223656,0.4685074815533062,0.5310776117248441,0.4747106030949773,200,0.6928057218186365,0.7284095430613645,0.7824124093499073,0.734117788134923,200,0.06545981601255141,0.09307498876917272,0.12078652786122779,0.09387293696692543,200,11.0,12.0,13.0,12.28 +fresh-zh-2,14,15,K,16,full_attention,200,0,200,0.46462449607206513,0.6070328595815226,0.9176525236340239,0.6582317644450814,200,0.08328344970941544,0.12113402411341667,0.3236641436815262,0.17590652227401735,200,5.281462830282894,8.308699378019789,10.48050251405668,8.067440658021955,200,0.3300914268926809,0.5192937111262368,0.6550314071285425,0.5042150411263722,200,0.035120035114892624,0.061715949077460135,0.13529919761295675,0.07381718642926435,200,0.16553471768371036,0.2276893460589769,0.36716577844031073,0.24986084540992035,200,0.515340575317463,0.6027966234087128,0.6947292687498511,0.6061065107187212,200,0.7731339627412669,0.8269406759418431,0.8733333931630004,0.8250958771375296,200,0.033964985073823624,0.04852964649217477,0.07672307216966229,0.05181347697354823,200,9.0,11.0,12.0,10.54 +fresh-zh-2,14,15,V,16,full_attention,200,0,200,0.43608817551285034,0.5877294522942975,0.7734893830027431,0.5942687213839963,200,0.08769998326897621,0.1380266696214676,0.26393826007843013,0.16245035119354725,200,5.665778145193463,8.085009761817142,10.301194964516224,7.995298297429854,200,0.35411113407459144,0.5053131101135714,0.643824685282264,0.4997061435893659,200,0.03690271985272979,0.06526534695629435,0.12160415428480058,0.07414630896374609,200,0.1771804094764731,0.26162794853663407,0.3634528346867949,0.2644792932033537,200,0.5191888591619367,0.590894168680582,0.6677577884978555,0.5926082526576955,200,0.7605760194967105,0.8007709284103918,0.8472835710348634,0.8034318327313571,200,0.03868931178641103,0.06046588427295689,0.08614074502141783,0.062023290527878014,200,10.0,11.0,12.0,11.12 +fresh-zh-2,14,15,QKV,16,full_attention,200,0,200,0.6588485824642702,0.7985384785570204,0.9420318270102144,0.8033623660623561,200,0.11081496700644493,0.15258733183145523,0.21193956583738327,0.15762553729116915,200,7.865406633211771,9.933724253112251,12.03145499226041,9.954708294043309,200,0.4915879145757357,0.6208577658195157,0.7519659370162757,0.6221692683777068,200,0.021989867739256368,0.04071173031049086,0.06894840633322505,0.04300444815259835,200,0.14215656997081155,0.18820013325602533,0.26432110425775074,0.1973612923287556,200,0.467392659436809,0.5320164441979365,0.5976185966363394,0.5322297243982639,200,0.7199498207634355,0.7742148498107582,0.8182871099556205,0.7707348312245728,200,0.05028904003853427,0.0762599505403896,0.10466600239162002,0.07766264171257939,200,11.0,12.0,13.0,11.72 +fresh-zh-2,14,15,JS,16,full_attention,200,0,200,0.01512076747858373,0.03115562235143443,0.06729762715403921,0.03770122548443737,200,0.004175087762996554,0.010222150478512049,0.03169021606445312,0.01672638167743571,200,2.5900783850252123,4.94621318500099,7.172146776592252,4.879790595054058,200,0.16187989906407577,0.30913832406256186,0.4482591735370158,0.30498691219087865,200,0.08205806728514975,0.14899000249238348,0.3451619100824118,0.1963934973272972,200,0.23590396148789686,0.35123879999721785,0.5772594587058741,0.3826099775251893,200,0.6510318304649428,0.7716097737657366,0.9082652232970603,0.7783450016384065,200,0.8792237458343642,0.9305443298912011,0.9751178867676571,0.929992078179312,200,0.004606125769961684,0.01396851584265563,0.030066682752376825,0.01600288792364242,200,4.0,7.0,9.0,6.895 +fresh-zh-2,14,15,sqrtJS,16,full_attention,200,0,200,0.40327895022928717,0.5365054130088538,0.7292719057761132,0.5568849647592288,200,0.06461491361260414,0.10110463947057724,0.17801222801208494,0.11628661843016744,200,6.907880354399288,9.898420046463862,11.712565489288654,9.519281348602632,200,0.4317425221499555,0.6186512529039914,0.7320353430805409,0.5949550842876645,200,0.024403667524334528,0.041095185414681816,0.08774647100017062,0.0545292239919708,200,0.14136737762319898,0.1893759313449372,0.28623818724530736,0.20715795888301936,200,0.4635272175302825,0.532743655826585,0.6539505442689656,0.5486530923641578,200,0.7286047244913315,0.7788100167488672,0.8538148259340307,0.7870064222228049,200,0.04632639967097977,0.07189789721739584,0.10056367831479326,0.07256751745330087,200,10.0,12.0,13.0,11.375 +fresh-zh-2,15,16,Q,8,linear_attention,400,0,400,0.24864988159388304,0.5586209930479527,1.0401263266801835,0.6069363517162856,400,0.05456886254251004,0.12792111933231354,0.2952774703502655,0.15306296054273844,400,5.770045212263006,6.717669513394278,7.29714490719177,6.638969277331126,400,0.7212556515328757,0.8397086891742848,0.9121431133989713,0.8298711596663908,400,0.013759924378304356,0.027269886414615967,0.05521026064645765,0.030706885065304793,400,0.18956146714875144,0.23293141593020006,0.31087134276634326,0.2412391608391436,400,0.3550328751768729,0.4028104585302471,0.46734903905943137,0.40598104465865803,400,0.6161506342212062,0.6610330547008453,0.7168799284119467,0.6652155509463867,400,0.10571194051484507,0.14007724562423712,0.17248942104426962,0.13932954589801697,400,7.0,7.0,7.0,6.945 +fresh-zh-2,15,16,K,8,linear_attention,400,0,400,0.3425259537994862,0.5605677645653486,0.9642518062144518,0.6109333288157359,400,0.06600420996546745,0.12057865411043167,0.23575374633073806,0.1364766418421641,400,6.284037060122268,7.037543929720813,7.441061326159853,6.94106453134625,400,0.7855046325152835,0.8796929912151016,0.9301326657699817,0.8676330664182812,400,0.010730778788841027,0.01953717890525121,0.03900956966782832,0.022623884629199082,400,0.17747353254130885,0.2099090813793001,0.2661498332253229,0.21698598720005893,400,0.33874312447757576,0.3787292206687007,0.4425003066188576,0.3850160825486165,400,0.6064605630188723,0.6473724201468627,0.7010343348116326,0.650374956677681,400,0.11369260031897471,0.1492940943319821,0.17510217681120963,0.14684269256281496,400,7.0,7.0,7.0,6.985 +fresh-zh-2,15,16,V,8,linear_attention,400,0,400,0.33883479349315165,0.49067915976047516,0.7781152866780758,0.537251534841489,400,0.0661492258310318,0.1042998768389225,0.16803258061408996,0.11695922539569438,400,6.407051522559302,7.017860494346861,7.362485155716969,6.925213526031905,400,0.8008814403199127,0.8772325617933576,0.9203106444646211,0.8656516907539882,400,0.012369951644084026,0.019992653703159502,0.035517775869395765,0.022994177009203982,400,0.17931915163200834,0.20665791209772116,0.2571470550933987,0.21400623366144877,400,0.34023314145529765,0.37792878914551137,0.4358666533560006,0.3839579797072345,400,0.6184365700382437,0.6490237659799967,0.698243795827725,0.654650834291582,400,0.11162342273857608,0.14300708343479834,0.16776378952868728,0.14137951810338809,400,7.0,7.0,7.0,6.945 +fresh-zh-2,15,16,QKV,8,linear_attention,400,0,400,0.38741979040205476,0.5657934974879026,0.910742022097111,0.6129876199923455,400,0.07668954432010651,0.12373138591647148,0.2402814731001854,0.13864466551691293,400,6.368793846914508,7.025786703013708,7.387370721666261,6.9435758599412205,400,0.7960992308643134,0.8782233378767135,0.9234213402082826,0.8679469824926526,400,0.011847039451803317,0.01980893527045586,0.03658934924168583,0.022339064833749908,400,0.182912063833162,0.21526690923081016,0.2632164789024179,0.2201973147615579,400,0.3437481148078461,0.37940296373262683,0.434492051867025,0.3843095921040423,400,0.6116607807300233,0.6449124359115901,0.6920372637657842,0.6487444832858305,400,0.11730154955556021,0.14904086442650377,0.1741916647309261,0.14751417544030165,400,7.0,7.0,7.0,6.9875 +fresh-zh-2,16,17,Q,16,linear_attention,400,0,400,0.09445429259212688,0.3330305354902521,0.6300873909145593,0.35232617048794057,400,0.013591364864259959,0.04826712794601917,0.09376184791326524,0.05331744477152824,400,10.161172040523313,12.080246441309388,13.366395755751512,11.846574030085183,400,0.6350732525327071,0.7550154025818367,0.8353997347344695,0.7404108768803239,400,0.013135449852026252,0.021631753441770426,0.038308155351018225,0.02470932494893005,400,0.11512093368636908,0.14511243305142957,0.19228778304674032,0.15156475087273022,400,0.3959751959537893,0.44891053428666305,0.5184686620837203,0.455973149025203,400,0.6783074816880845,0.7259297800612412,0.7787749747757304,0.7271356315444375,400,0.07351917756231743,0.09849444816016227,0.12295651832947312,0.09871549142563925,400,12.0,12.0,13.0,12.4075 +fresh-zh-2,16,17,K,16,linear_attention,400,0,400,0.3555954312905669,0.4691151543520391,0.6366063117049635,0.48882711899990683,400,0.045802701637148856,0.06492159515619278,0.09854218289256098,0.06984606887679547,400,10.784806195149393,12.291668252787948,13.526664860218462,12.202059169589552,400,0.6740503871968371,0.7682292657992468,0.8454165537636539,0.762628698099347,400,0.012189925442819327,0.020112992031461552,0.03223792389198978,0.021571073580244637,400,0.11196684740687264,0.13800857600295957,0.18228591974122593,0.14327362168769173,400,0.39404247803126224,0.4414605419003263,0.49904920023268706,0.44402687988877376,400,0.671489071663334,0.7166874732675121,0.76115670179838,0.7166256876029167,400,0.08100476492872255,0.1029005910102415,0.12740614907155628,0.10303927259724248,400,12.0,13.0,13.0,12.5425 +fresh-zh-2,16,17,V,16,linear_attention,400,0,400,0.30306056414265187,0.43522794684395194,0.6135100118350238,0.4507261173709412,400,0.0417540330439806,0.06939461827278137,0.10665249302983285,0.07370163979008794,400,8.827249157789842,11.65355058249649,13.07479970261715,11.268648592961451,400,0.5517030723618651,0.7283469114060306,0.8171749814135719,0.7042905370600907,400,0.01491521048992218,0.024864808512287267,0.05417150642889014,0.030883704140377846,400,0.11985205786177057,0.15033578757521107,0.22751606588936885,0.16358233988972515,400,0.4063767005069789,0.4665859689745926,0.5659985660339018,0.47852696574711495,400,0.6865106069309234,0.7427978399983297,0.8101035524571346,0.7450460448412782,400,0.058907354068096426,0.09283249991209644,0.11678752802122565,0.0904307666997741,400,11.0,12.0,13.0,12.1075 +fresh-zh-2,16,17,QKV,16,linear_attention,400,0,400,0.3363911425229162,0.4552599131129682,0.5883902822155506,0.46049709479673767,400,0.04440656490623951,0.06268422305583954,0.0890053138136864,0.06520735373720526,400,10.640427424213355,12.405638203641868,13.49641390222488,12.231781648517819,400,0.6650267140133347,0.7753523877276167,0.843525868889055,0.7644863530323637,400,0.012366721083767839,0.01931574322495353,0.03358005081346356,0.02144345314865029,400,0.11355263461077202,0.1357367886467502,0.1761219271462543,0.1421765764855809,400,0.3914154684341705,0.43695698272678124,0.5046466809379332,0.4433029364732691,400,0.6703475422451087,0.7152187196005687,0.7654984361556008,0.7173421133521961,400,0.0798484359289908,0.10357157794877164,0.12668348949891034,0.10374395964444898,400,12.0,13.0,13.0,12.555 +fresh-zh-2,17,18,Q,16,linear_attention,400,0,400,0.1226691393298097,0.36147017194889486,0.5818089807871729,0.3540648076915386,400,0.024929285049438477,0.08050834760069847,0.13572832942008975,0.08136557938531042,400,6.93896881298955,9.537098674929744,11.661970203117695,9.402233284212386,400,0.4336855508118469,0.596068667183109,0.728873137694856,0.5876395802632741,400,0.024798724534623597,0.045177272200546376,0.08705875631654728,0.051768609571302644,400,0.1705293584592322,0.22210662033084488,0.31747853930100234,0.23201714022243503,400,0.45089993313094884,0.5184210730765397,0.6200332761872304,0.5269981441938874,400,0.6974683834354041,0.7461505028593447,0.8144960123799418,0.7511065096398988,400,0.060643572256052714,0.08957196532466774,0.12198213765927611,0.09095809517026986,400,11.0,12.0,13.0,12.1 +fresh-zh-2,17,18,K,16,linear_attention,400,0,400,0.19490073614288125,0.3274110041093081,0.4664688227698207,0.3297938077621802,400,0.04472722373902798,0.07343719154596329,0.10741687789559365,0.07637801371514798,400,7.555860961928367,9.318749955493983,11.107362382531626,9.346887474973633,400,0.47224131012052295,0.5824218722183739,0.6942101489082266,0.584180467185852,400,0.029365849699411545,0.04779808704484728,0.07450409766001101,0.04998138141232276,400,0.18414481635182506,0.23334921275842735,0.2806722098457578,0.23451272626257072,400,0.46357790736603127,0.5168493836462402,0.5968661023740877,0.5242895199027334,400,0.7029334359671282,0.7424617956157007,0.8019086896726701,0.7467872524102458,400,0.06657680915799008,0.09377956788172392,0.12029771212046857,0.09337794569345953,400,11.0,12.0,13.0,12.235 +fresh-zh-2,17,18,V,16,linear_attention,400,0,400,0.2934502001851797,0.39912510151043534,0.5174729821388611,0.4047692112044024,400,0.09749863445758819,0.12414421513676643,0.15401251018047332,0.12416316639166325,400,5.038564560821192,6.84185470418899,8.95978173056595,7.007019991857028,400,0.3149102850513245,0.4276159190118119,0.5599863581603719,0.43793874949106426,400,0.052383852892791564,0.08923647929204578,0.14503411984649814,0.09236964792966995,400,0.2409126134233609,0.30608287171527376,0.40377312461326387,0.31315867245676465,400,0.5254419071946036,0.5991636954591414,0.6798072659377953,0.6003078898882682,400,0.7443839611331483,0.7915122901305436,0.8450080838566334,0.7936948160322159,400,0.0487087549708656,0.07587324789221103,0.09884728461044955,0.07519609760455154,400,10.0,12.0,12.0,11.52 +fresh-zh-2,17,18,QKV,16,linear_attention,400,0,400,0.29398879918735477,0.3750228965654969,0.48529430138878527,0.3857895018589625,400,0.07874351963400841,0.09750906378030777,0.12123734056949616,0.09934341276064515,400,6.951435027987448,8.432224500016428,10.146284632487387,8.514828667233033,400,0.4344646892492155,0.5270140312510267,0.6341427895304617,0.5321767917020646,400,0.03846234967817875,0.05983218995178738,0.08677887056518972,0.061287499371776875,400,0.2139946033890815,0.2603432486226124,0.31297290727257504,0.26096849136490174,400,0.48203587586463564,0.5411831537102381,0.6186965811506501,0.5462950208160391,400,0.7173549642805743,0.7548581887216801,0.8130010259623329,0.7600862523848175,400,0.0622795898222382,0.08835012463963018,0.1149141783751941,0.0882475123247091,400,11.0,12.0,13.0,12.05 +fresh-zh-2,18,19,Q,16,full_attention,200,0,200,0.40503120101056994,0.5028677717782557,0.612515045888722,0.505402263617143,200,0.05769512243568897,0.08007610216736794,0.10372484102845192,0.08085989257320762,200,10.48549202320829,12.08531815382366,13.132081825304432,11.958726511611918,200,0.6553432514505181,0.7553323846139788,0.820755114081527,0.7474204069757449,200,0.014559355192104564,0.021594730705182065,0.0350611950326472,0.023274466726017694,200,0.1318316271507269,0.15768979903170172,0.19497668503474405,0.15998315774759458,200,0.4143940693475607,0.45017401534471846,0.507595150134935,0.4552125765543622,200,0.6658417785989769,0.706894163471876,0.7455587623980972,0.7047817209291677,200,0.10244972282154999,0.1205462444205549,0.14435799317186915,0.12263518954948081,200,13.0,13.0,14.0,13.075 +fresh-zh-2,18,19,K,16,full_attention,200,0,200,0.24221688634715974,0.31452087173238397,0.4046083310618996,0.3193610634095967,200,0.045973315462470056,0.06273973360657692,0.09917145818471906,0.06787015929818153,200,8.036370622749583,10.014142860527388,11.495391997186315,9.766300901367732,200,0.502273163921849,0.6258839287829617,0.7184619998241447,0.6103938063354832,200,0.02612420621532505,0.03985277323767919,0.0660781553099645,0.048112532343058255,200,0.16333885802039474,0.1983673422972086,0.2591638887113507,0.21397288050852384,200,0.47597161670898996,0.5317482705366685,0.5930921133800963,0.5370099012204113,200,0.7198316678216097,0.7578224563138349,0.7912789184585522,0.7589081603063189,200,0.08314563099626963,0.10030959633138392,0.12085481682745897,0.09949625474526165,200,12.0,13.0,13.0,12.5 +fresh-zh-2,18,19,V,16,full_attention,200,0,200,0.219464371772483,0.2803905897308141,0.4177178148180246,0.3227898630872369,200,0.043733523413538936,0.053726617246866226,0.06885573938488958,0.05766161188483238,200,10.102204406133366,11.490776082827278,13.111335719139854,11.522210342292077,200,0.6313877753833353,0.7181735051767049,0.8194584824462409,0.7201381463932548,200,0.014687947408494988,0.02616198204898496,0.038921239336181926,0.026724411572927237,200,0.15024668851850015,0.1849362979756537,0.21859909014579687,0.18295310416096044,200,0.4032218916229758,0.45713703207054,0.5141414231586733,0.45774826331708107,200,0.6458577812228684,0.684933156698807,0.7367127613882485,0.6901858682662803,200,0.1066450601564698,0.13335088715478763,0.15786431375570556,0.13350750822035157,200,13.0,13.0,14.0,13.32 +fresh-zh-2,18,19,QKV,16,full_attention,200,0,200,0.3294968142639846,0.3965512770228088,0.4856201363261789,0.4079517453722656,200,0.04950256980955601,0.061345502734184265,0.07889391258358952,0.06428984832018614,200,10.639726795051333,12.040430001644602,13.003814968129394,11.896175732781762,200,0.6649829246907083,0.7525268751027876,0.8127384355080871,0.7435109832988601,200,0.015360558173858537,0.021923786352624507,0.03358655348373444,0.02352446999362818,200,0.13557158666803165,0.15121115195854967,0.19280496233177397,0.1576780838303767,200,0.4227330702463845,0.4588547213598372,0.5105609907031137,0.46196181567596517,200,0.666401987984752,0.7140073510868454,0.7397872355751911,0.7096373145971054,200,0.09936859000570922,0.1181786828259131,0.14117991197913995,0.12074725231935556,200,12.0,13.0,14.0,13.01 +fresh-zh-2,18,19,JS,16,full_attention,200,0,200,0.0032219047937360303,0.008727351967536379,0.014426813827594741,0.008820296328706831,200,0.000962602801155299,0.003277353127487004,0.0074657529126852746,0.0038093298792955467,200,2.507341334910479,4.477504812221616,6.281203520584329,4.464567491200771,200,0.15670883343190495,0.279844050763851,0.39257522003652057,0.2790354682000482,200,0.10315228033370576,0.17156138441682822,0.35876087690123126,0.20615982106380534,200,0.2601172774806182,0.3719046340678134,0.6014802552390575,0.4050797685287161,200,0.6989415753564249,0.8121332768210971,0.8872430146730127,0.800883430708973,200,0.9016709446595074,0.9414718953816446,0.9677019058950461,0.936942245580977,200,0.0074748144061829395,0.01428673427781333,0.02403330531498279,0.01530508423302874,200,5.0,7.0,8.0,6.69 +fresh-zh-2,18,19,sqrtJS,16,full_attention,200,0,200,0.1801991507643834,0.27761521888896823,0.3554695088416338,0.27363203835324384,200,0.031025007553398612,0.05724814906716347,0.08640452027320862,0.05799578634090721,200,7.334868986779317,9.412911961385927,11.077758879926002,9.36214126871173,200,0.4584293116737073,0.5883069975866204,0.6923599299953751,0.5851338292944831,200,0.029622701960780155,0.046653341153934144,0.0787574263658076,0.050307644538884484,200,0.15641748139966435,0.1969667030021279,0.28189193173812366,0.20965446579842933,200,0.4930179417021164,0.5578293720615657,0.6167904546741356,0.5545430430302752,200,0.7463542328960098,0.7865632191004863,0.8257781520688219,0.7867799748411661,200,0.06078319793718209,0.07615962590512901,0.09195109644556287,0.07667649122156311,200,11.0,12.0,12.0,11.6 +fresh-zh-2,19,20,Q,8,linear_attention,400,0,400,0.21508374549448492,0.5137318698689342,0.8095862653106451,0.5110034689609892,400,0.05792464315891266,0.13033463805913925,0.27519477903842926,0.14895880213938653,400,4.3819586456215625,6.017533519381162,6.8519280520768895,5.797861515507205,400,0.5477448307026953,0.7521916899226453,0.8564910065096112,0.7247326894384006,400,0.023936374496984693,0.047064097325120686,0.11795251774476971,0.06080436203703736,400,0.20859178786860513,0.27265141899623846,0.39654388543955693,0.2894650706168785,400,0.3791282476318722,0.45449854615605156,0.5681392614664926,0.4675485302617241,400,0.655418862758822,0.7193698040441492,0.8104280062026928,0.7283010855111249,400,0.06287880287561806,0.10257799040047749,0.13276625473307394,0.10049228189793982,400,6.0,7.0,7.0,6.475 +fresh-zh-2,19,20,K,8,linear_attention,400,0,400,0.3427822642959654,0.5048338016495109,0.7881359469145537,0.5454670517286285,400,0.0829150527715683,0.14131422340869904,0.23521096855401996,0.15315406411886215,400,4.911879786919975,5.901373501250659,6.823242737487929,5.8532807064405805,400,0.6139849733649969,0.7376716876563324,0.8529053421859911,0.7316600883050726,400,0.024637585478146127,0.05080252054429725,0.08981635383197598,0.056645582446445644,400,0.20378490756400489,0.27358861276508073,0.36690619572963945,0.2799069249472273,400,0.3829667237758037,0.46298412617076984,0.5406730375392912,0.4646270621183692,400,0.6674417390852442,0.7312380982647706,0.7885197384753848,0.730605152188823,400,0.07488810289498735,0.09869312354057228,0.12902218406422897,0.10048224958950708,400,6.0,6.0,7.0,6.4425 +fresh-zh-2,19,20,V,8,linear_attention,400,0,400,0.32972691711038354,0.48440330661833286,0.6789735980331898,0.49958652374509255,400,0.09193616211414338,0.12836939841508865,0.17157514542341232,0.13098743319511413,400,5.000756979560625,6.051076668892893,6.789549280380749,5.957479040345568,400,0.6250946224450781,0.7563845836116116,0.8486936600475936,0.744684880043196,400,0.025468781366201745,0.046011273704211014,0.08567988267912352,0.051665490547663,400,0.21128526954801474,0.26144100848979157,0.3395312924942187,0.2691798739672016,400,0.39506932309986503,0.45706279089909957,0.5394562271356179,0.46124843155852474,400,0.6641761654777416,0.7295426248186352,0.7866718082456551,0.7268695616593093,400,0.07619679033640862,0.10085283293724095,0.13390191581457864,0.10338244059518958,400,6.0,7.0,7.0,6.51 +fresh-zh-2,19,20,QKV,8,linear_attention,400,0,400,0.38525272272527217,0.5362839074805379,0.7381349332630635,0.5451388669759035,400,0.0965897113084793,0.14157231897115707,0.213453708589077,0.15015659496188163,400,4.922100649057225,6.078426748611697,6.7823487381929155,5.964063396523909,400,0.6152625811321532,0.7598033435764622,0.8477935922741144,0.7455079245654886,400,0.025647498001710387,0.04516144218484122,0.08933190147795704,0.05227826199450467,400,0.21631362388828146,0.26545717202408103,0.3576051413367827,0.27600571663142187,400,0.3838206674834326,0.45324502750876,0.5359666313199962,0.45756213040415333,400,0.661495287235009,0.7211589259372411,0.7836710108285425,0.7217076288292066,400,0.07647427260789201,0.10371297876202508,0.13281400260502527,0.1039282969644982,400,6.0,7.0,7.0,6.54 +fresh-zh-2,20,21,Q,16,linear_attention,400,0,400,0.10304542187368497,0.18540286284405738,0.5109435369260609,0.2405839985096827,400,0.015901263803243637,0.026190352626144886,0.08744883015751839,0.037909609344787897,400,10.040861265204287,11.913571146703452,13.123788089770242,11.710052273507058,400,0.6275538290752679,0.7445981966689658,0.8202367556106401,0.7318782670941911,400,0.01461068294478387,0.02286708073796547,0.03956611239332996,0.025503104133089014,400,0.12111623131238412,0.14767706658981455,0.19178790608156493,0.15300698989277303,400,0.41199169714094985,0.46308137735745003,0.5349818250884552,0.4685322732736497,400,0.6863506290532629,0.7255715115350507,0.7785222291004409,0.7293508652428403,400,0.0776104582270633,0.10078321164690446,0.12054050159329259,0.09975993820457407,400,12.0,13.0,13.0,12.4725 +fresh-zh-2,20,21,K,16,linear_attention,400,0,400,0.21153694412205368,0.3370554861612618,0.4722403896506876,0.347156229307584,400,0.03031688090413809,0.05145837925374508,0.07830771803855897,0.055411372890230265,400,9.879459254420492,11.682024705390262,12.868192743014005,11.515045769575977,400,0.6174662034012808,0.7301265440868914,0.8042620464383753,0.7196903605984986,400,0.016225056484297035,0.024641711250146366,0.04130150171518798,0.027227095213540133,400,0.12431170982791663,0.14888633517991134,0.20967343368917718,0.15904126477403646,400,0.4213490816677721,0.47204729710719506,0.5324938352650782,0.4741402249553876,400,0.6942610972930997,0.7318381445513076,0.7681195633640776,0.7322326672854055,400,0.08223156982562664,0.09974593787774501,0.11683251668170767,0.09970676913028458,400,12.0,12.0,13.0,12.4525 +fresh-zh-2,20,21,V,16,linear_attention,400,0,400,0.25448002079501747,0.33734766487032175,0.48101746896281844,0.35774384666641706,400,0.03685517683625221,0.05452725104987621,0.10371628552675248,0.06208037775475532,400,8.485617938460706,11.498552572886286,12.962185097254881,11.005043368961585,400,0.5303511211537941,0.7186595358053929,0.8101365685784301,0.6878152105600991,400,0.0156239864572188,0.026098635827590207,0.0590373200550462,0.034194447478901394,400,0.1232091361837478,0.15588210786453166,0.24859857978566055,0.17349321067389994,400,0.41800668150827985,0.4789534372257378,0.5785416960894778,0.49080687118265415,400,0.6894330266362995,0.7371233314608696,0.7955036321677438,0.7407626795801121,400,0.0723536338515867,0.09488413526003528,0.11590655219014549,0.09447652341728534,400,11.0,12.0,13.0,12.29 +fresh-zh-2,20,21,QKV,16,linear_attention,400,0,400,0.24738598219119012,0.32032593665644526,0.4463034125044942,0.33773504111915825,400,0.036656650528311734,0.05000334046781063,0.0799660623073578,0.05475362613331527,400,9.628383600355354,11.612754879894869,12.818492427825559,11.384977532329312,400,0.6017739750222096,0.7257971799934293,0.8011557767390974,0.711561095770582,400,0.016546447079225247,0.025186361613840233,0.04411692730333632,0.028426675120830858,400,0.1248618433701965,0.15305463347382775,0.2206374854934659,0.16339800769931917,400,0.4274384049423568,0.4723972098022552,0.5398562534436137,0.4782124330546534,400,0.6958397222095623,0.7313340856922378,0.7726626289521457,0.7333113211598637,400,0.08060534832717549,0.09896946954288474,0.11601362660290662,0.09913390190415494,400,12.0,12.0,13.0,12.4425 +fresh-zh-2,21,22,Q,16,linear_attention,400,0,400,0.06882400247268379,0.18573116953484714,0.3417895504739136,0.19976254678345867,400,0.008275804482400418,0.021871909499168396,0.05014982260763649,0.03415667988359928,400,11.53338278844459,13.792799943032323,14.559627926701252,13.235170147817184,400,0.7208364242777868,0.8620499964395202,0.9099767454188282,0.827198134238574,400,0.006595294774001043,0.010668368357011362,0.025819334935722433,0.019645070015268003,400,0.10079100224811331,0.12300982419171573,0.1785470977113007,0.14065775531096097,400,0.3536825296093656,0.38913870660580163,0.4563822598814049,0.40233657768886316,400,0.6217019126771438,0.6487547887609069,0.7124549021261866,0.6599026439763519,400,0.10609889149215687,0.14534908394769802,0.16286836837277346,0.13977136511469054,400,13.0,14.0,14.0,13.47 +fresh-zh-2,21,22,K,16,linear_attention,400,0,400,0.12104345322586596,0.23283731169067323,0.3578012593556196,0.23653698855865513,400,0.015482682920992377,0.027707722038030624,0.04692872315645219,0.030609547620406373,400,12.209475786281681,13.718788301032,14.654972039056808,13.54949104198799,400,0.7630922366426051,0.8574242688145,0.9159357524410505,0.8468431901242494,400,0.006118642885511519,0.01108558538043812,0.02069747750651923,0.012538671388088356,400,0.09938112874720664,0.12272737716083175,0.1707865146552365,0.12908202086747686,400,0.3535306625396771,0.3903328504248249,0.43731792371885814,0.39327674229700643,400,0.6187657935730863,0.6551218826756682,0.6858540984771537,0.6538674303318862,400,0.1184405240862148,0.14566001010113805,0.16434477076615162,0.14354063043055493,400,13.0,14.0,14.0,13.5925 +fresh-zh-2,21,22,V,16,linear_attention,400,0,400,0.15004875496961176,0.22157417214475572,0.3262884737923742,0.2326084670861019,400,0.017553484439849852,0.029581760056316853,0.04784855768084526,0.03201089103007689,400,11.474212133810969,13.439853596161646,14.347590617520165,13.093910583757301,400,0.7171382583631856,0.8399908497601029,0.8967244135950103,0.8183694114848313,400,0.007677988259530112,0.01269927728211313,0.026295530734713204,0.01609332024515764,400,0.10530787981043532,0.12665281092913822,0.18835361687215538,0.13892302081483077,400,0.359531339034474,0.40470571095549857,0.46519365299238413,0.4092468009260294,400,0.6326578863228246,0.6633027289735123,0.7059341314523627,0.6663453437999372,400,0.11156727204679956,0.1390167564476057,0.15730004544676104,0.13643011966573246,400,13.0,13.0,14.0,13.405 +fresh-zh-2,21,22,QKV,16,linear_attention,400,0,400,0.15914542502723633,0.22510805120691657,0.31633296282961965,0.23658636618521997,400,0.019032919034361842,0.028000143356621265,0.04804670289158821,0.0344468352664262,400,11.95667753916341,13.745969464986537,14.598963630333644,13.352497291946415,400,0.7472923461977131,0.8591230915616586,0.9124352268958528,0.8345310807466509,400,0.006397880505664693,0.010931853868594639,0.022544294756478046,0.01578307453826965,400,0.09887806792057371,0.12466353348354708,0.17719463457939236,0.13631433232841927,400,0.35319149936210925,0.3914826475911559,0.44710681105569816,0.3982467478053253,400,0.6238262192048054,0.6514924336106952,0.6909869701161078,0.6558849388771258,400,0.11863972786289936,0.14608225758231444,0.16280632146758137,0.14293468858867048,400,13.0,14.0,14.0,13.59 +fresh-zh-2,22,23,Q,16,full_attention,200,0,200,0.3663093815790489,0.4459937726496719,0.5909654686227441,0.4641306163536501,200,0.05626216419041157,0.08478598669171333,0.15513627529144283,0.09579937724396587,200,8.060869493090234,10.166767162297202,11.919316132141498,10.090913186669624,200,0.5038043433181396,0.6354229476435751,0.7449572582588436,0.6306820741668515,200,0.022823940753937108,0.038250331330052637,0.06566540717310967,0.041919581722297344,200,0.1377878541184072,0.1914478746767771,0.266630418197491,0.2001405225405304,200,0.4477244572239665,0.5155118295324064,0.5807146420264798,0.5139650922066674,200,0.7213904604665722,0.7567208381152651,0.7943420764739438,0.7572643605811561,200,0.05940194695495365,0.08243452363013759,0.1020097427749334,0.08178921648097236,200,11.0,12.0,13.0,11.83 +fresh-zh-2,22,23,K,16,full_attention,200,0,200,0.34670295999385425,0.4595507131598424,0.640807054657489,0.4635155391111039,200,0.09416036754846574,0.14216911792755127,0.213063508272171,0.151237925440073,200,5.35434945996427,6.708372923294132,8.513620827933241,6.787457748583879,200,0.3346468412477669,0.4192733077058832,0.5321013017458276,0.42421610928649245,200,0.05862300032790808,0.09233903515062016,0.13255331528135744,0.09608886765725147,200,0.23888827508493102,0.3292894637945016,0.39130248330263434,0.32315702193155876,200,0.552844368582166,0.5964123074210952,0.6557452330915107,0.5972552385323177,200,0.7834214335094696,0.8121461600193804,0.840694013400649,0.8117203352348517,200,0.04119351933710362,0.062157479351489575,0.07525054107634731,0.06008881955618765,200,10.0,11.0,12.0,11.0 +fresh-zh-2,22,23,V,16,full_attention,200,0,200,0.23345311305602084,0.2993957453290932,0.4970016268664039,0.33736590946442446,200,0.0518654689192772,0.10119126364588737,0.25077765882015224,0.13101759046316147,200,3.4331677399617866,5.930392238714632,8.49810179097805,5.862621956199848,200,0.21457298374761166,0.3706495149196645,0.5311313619361281,0.3664138722624905,200,0.05885211205675477,0.11319788923692634,0.24417404771421825,0.14043483524261244,200,0.22221848318885984,0.3428125720414256,0.5099584990605941,0.36012249946296426,200,0.5889063731709081,0.6756813694430817,0.7790410184439118,0.6834247171073744,200,0.8077367068851568,0.8538294752473312,0.9010914356878797,0.8527344583989407,200,0.03092599917273619,0.04640051538915988,0.06807046784921209,0.04891952940709511,200,8.0,10.0,11.0,9.96 +fresh-zh-2,22,23,QKV,16,full_attention,200,0,200,0.33836357791442423,0.43763377936556935,0.5444308941019699,0.444455892132537,200,0.07637957483530045,0.1244831420481205,0.21034854501485822,0.13109229957684876,200,5.704836845565464,7.681773575478758,9.900431518716248,7.692501569460188,200,0.3565523028478415,0.48011084846742236,0.6187769699197655,0.4807813480912618,200,0.0410727895256019,0.07219018256859389,0.12030961605504087,0.07904438930688229,200,0.20460037665902386,0.28543954657863846,0.3743948821036761,0.2877652193445346,200,0.5205230142876641,0.5828079036475049,0.6447792654653233,0.5822370229148761,200,0.7659429372615634,0.7960861698590243,0.8323762151950365,0.7969292281607114,200,0.045599628145957256,0.07020308719780469,0.08575859071507999,0.06773132437934061,200,10.0,11.0,12.0,11.29 +fresh-zh-2,22,23,JS,16,full_attention,200,0,200,0.0007367950744956887,0.00772236482562505,0.01609503587471863,0.00822785697112763,200,0.00028562566149048507,0.0036876266822218895,0.009380965214222669,0.004447453631291865,200,1.9799851413106009,2.9869010799867652,4.352054934982288,3.102468910118243,200,0.12374907133191256,0.18668131749917283,0.272003433436393,0.1939043068823902,200,0.1784295163937018,0.29044817210863744,0.47205792347631015,0.3141883179825297,200,0.34939824391502955,0.4899743034386207,0.6879486427401185,0.5101724534354644,200,0.8297345313004074,0.9041209886600927,0.9574739533921884,0.8961743464279366,200,0.9497207380272148,0.9759621593939646,0.9900062278066352,0.9725891211013746,200,0.0013486610062881663,0.0042507848975992,0.011009112195801963,0.005332989943601728,200,3.0,4.0,6.0,4.42 +fresh-zh-2,22,23,sqrtJS,16,full_attention,200,0,200,0.07260303560906324,0.23694017919478938,0.30805367498251146,0.21687023622929702,200,0.016900292597711086,0.060725798830389977,0.09685527831315995,0.06006529964040965,200,5.325534666811308,7.062343284874409,8.729592759987023,7.017189744398395,200,0.33284591667570673,0.44139645530465055,0.5455995474991889,0.43857435902489966,200,0.05552307967244228,0.08436913727368543,0.13362625792397423,0.09300369715738753,200,0.20715641115806013,0.2658266403035652,0.3515270913709329,0.27525113136498064,200,0.5768886759222794,0.6532001293957612,0.7380530980760734,0.6556035301474417,200,0.8010733283871241,0.8517030914115188,0.8938012135357822,0.8501187743965459,200,0.0280943046160524,0.046067984663694855,0.06711216303038915,0.04665465452250071,200,9.0,10.0,11.0,10.015 +fresh-en-1,0,1,Q,16,linear_attention,400,0,400,2.234930998273194,3.5310883498750627,4.6527290633879606,3.469649436067848,400,0.41280797123908997,0.8332870602607727,1.209194898605347,0.8354474094882608,400,6.226367467443348,8.139899293210867,10.510365405652692,8.18054595445664,400,0.38914796671520924,0.5087437058256792,0.6568978378532933,0.51128412215354,400,0.034820814169823515,0.06437513148421117,0.10464783173077424,0.06911764366069677,400,0.16843947008601867,0.23012941521497365,0.32827907476462276,0.240249907140078,400,0.507709366121756,0.6023417748356543,0.6949911487259492,0.6030826432864502,400,0.7521558717432947,0.8138294155386356,0.8701449698306754,0.8112156140811752,400,0.03815905125688387,0.06297055750476736,0.09611574582932063,0.06490486028917865,400,10.0,11.0,12.0,11.0175 +fresh-en-1,0,1,K,16,linear_attention,400,0,400,2.6672797936946155,3.425931252539158,4.5168875466100875,3.524958591017639,400,0.489157047867775,0.6667049527168274,0.9598093152046206,0.7105951488763094,400,7.773096798822127,9.21570544596814,11.478127920136487,9.411324637121325,400,0.48581854992638296,0.5759815903730088,0.7173829950085304,0.5882077898200828,400,0.026263782535248418,0.04907777647301703,0.07055878795953471,0.04951417146096089,400,0.1536270424935471,0.1943972618510333,0.2586597019235089,0.201790065585419,400,0.4702001079767811,0.5670607842769645,0.6468327655968112,0.5600256623955385,400,0.7240993364085627,0.7893527719767768,0.8405807624921381,0.7852669490279997,400,0.048173260955288955,0.07166116040510198,0.0979250143869214,0.072823507184225,400,10.0,12.0,12.0,11.4825 +fresh-en-1,0,1,V,16,linear_attention,400,0,400,1.6181118641979992,2.6892965445294976,4.556001805514098,2.970108881509077,400,0.2978472948074341,0.5580625236034393,1.102718198299408,0.677044918499887,400,6.352610433718244,8.778607005749569,11.404983940324158,8.815547892286256,400,0.39703815210739024,0.548662937859348,0.7128114962702599,0.550971743267891,400,0.02685969798896445,0.054840842425672894,0.10124352178696013,0.06065345569593322,400,0.15031578023950862,0.21015335814611086,0.30510330548762044,0.22146373954243845,400,0.4689738279110144,0.5706814381429993,0.6804917009819806,0.5767158032949978,400,0.7385965063342251,0.8096059756201381,0.8721097435170654,0.8067382010773199,400,0.03293960185396647,0.06087632626720197,0.0882869201475248,0.061249493470910375,400,9.0,11.0,12.0,10.9375 +fresh-en-1,0,1,QKV,16,linear_attention,400,0,400,2.734647191502154,3.3793744368012995,4.43082140609622,3.520445009493269,400,0.5053462266921998,0.7177450358867645,1.01717768907547,0.7542269672453403,400,7.593485629853883,8.982527779648695,10.561354533006119,9.036272746571104,400,0.4745928518658677,0.5614079862280434,0.6600846583128824,0.564767046660694,400,0.03433048089562333,0.05208238818291653,0.07380461636970538,0.053516467141441114,400,0.16797489114982522,0.20667639999959206,0.2679313288961226,0.21318120192819542,400,0.49818285974571835,0.5755689018982981,0.6515263212168357,0.5746907963534565,400,0.7351683944811394,0.7969288402708317,0.8450067977184706,0.7925291609314834,400,0.04653508246529315,0.07042835489747537,0.09824510164814938,0.07151269004665935,400,10.0,11.0,12.0,11.4425 +fresh-en-1,1,2,Q,16,linear_attention,400,0,400,0.4932698825286934,1.0213602213189006,1.6281458265148103,1.0659803825702694,400,0.08536823317408562,0.20087481290102005,0.38239128291606905,0.22786430266685784,400,7.016876538589014,9.213822806373706,11.164212981295533,9.181035817026496,400,0.4385547836618134,0.5758639253983566,0.6977633113309708,0.573814738564156,400,0.028876716779932716,0.04910184659778931,0.08534791137694102,0.053861621600688706,400,0.14451681620107873,0.19255139381257563,0.2763586186224678,0.20400164197210358,400,0.4788120046845227,0.5490563135957811,0.6429579124172133,0.5532461573490183,400,0.7493208500239771,0.8154344096519979,0.8899122047395697,0.818275563824142,400,0.01821186446497532,0.052334861114869034,0.08013902218047589,0.051337913255023085,400,9.0,11.0,12.0,10.4925 +fresh-en-1,1,2,K,16,linear_attention,400,0,400,0.7984649914782495,1.1629895486403257,1.7354826693423095,1.2208493195035408,400,0.11778525412082673,0.19534742832183838,0.4292791157960893,0.24494460735470056,400,7.695978599585112,9.888124659408357,11.83680499946064,9.796729351936944,400,0.4809986624740695,0.6180077912130223,0.73980031246629,0.612295584496059,400,0.02344774271655922,0.04120684031683605,0.07193403343239543,0.046350168567002754,400,0.13253593648974718,0.17377125547815958,0.26944248566215007,0.19146204771406913,400,0.4477017522745427,0.5198989814740866,0.6051887150208005,0.5232734592764924,400,0.7355654627186871,0.7908391563493042,0.8813063391472371,0.7972375229561662,400,0.02322053734091502,0.060272258327300224,0.09060879268146069,0.05924438361702439,400,9.0,11.0,12.0,10.87 +fresh-en-1,1,2,V,16,linear_attention,400,0,400,0.5248587501700968,1.0819192812778056,1.9770300139207393,1.1859704301759484,400,0.09696745052933693,0.20412401109933853,0.43642934262752536,0.24929563925601542,400,7.0074239445394175,9.114729800784406,11.391004846100728,9.148809454012484,400,0.4379639965337136,0.5696706125490254,0.7119378028812955,0.5718005908757803,400,0.02697466606518288,0.050360024193146447,0.08555317096717797,0.05513230238403486,400,0.1387922507717615,0.19145725407235945,0.2817647128892312,0.2037644935932714,400,0.46503744103682637,0.5561711120190782,0.6458764365310309,0.5563113581367254,400,0.7437048128449286,0.822518016356446,0.9070976014349672,0.8222166396198135,400,0.017000051400460736,0.049588532382416464,0.08641712655785642,0.05027575701830296,400,8.0,11.0,12.0,10.3875 +fresh-en-1,1,2,QKV,16,linear_attention,400,0,400,0.7995891033613589,1.1612361561274156,1.7709278389811516,1.2402550661597342,400,0.1203035831451416,0.2050672098994255,0.3727079629898072,0.23898614736273885,400,7.963857490650867,9.888763036555027,11.841037769838795,9.881150120146764,400,0.49774109316567916,0.6180476897846892,0.7400648606149247,0.6175718825091727,400,0.023415603549033437,0.0411998762044576,0.06727178425682435,0.04454462898444515,400,0.13096510156449193,0.17470327702462282,0.25301981863755385,0.18590808871671968,400,0.4453974403840927,0.5183547634691132,0.591629820201727,0.5212573713438974,400,0.7280081183938285,0.7906337811310362,0.8790847538668738,0.7978887665102706,400,0.022839348819696004,0.06085974901084702,0.09160044560494329,0.05911894241895917,400,9.0,11.0,12.0,10.865 +fresh-en-1,2,3,Q,16,full_attention,200,0,200,0.948799574363511,1.3145958668319508,1.764732945081778,1.3375451209355378,200,0.26402898132801056,0.3268849402666092,0.41940989196300504,0.3360720041394234,200,6.070270692698609,7.955288576687888,10.616733320086226,8.230375248220831,200,0.3793919182936631,0.497205536042993,0.6635458325053891,0.5143984530138019,200,0.033803723033825506,0.06741639882994804,0.10905313187222014,0.06943344719137264,200,0.1841855079953235,0.2545527535195912,0.34150651751935435,0.2594988540111153,200,0.4796707066968893,0.570146309923004,0.6625415744944808,0.5701600312163876,200,0.7513670141266213,0.8197962962918505,0.9033098802201723,0.8209732150259172,200,0.01245702187643904,0.045927958651912064,0.0806873403515382,0.046497614439233226,200,8.0,11.0,12.0,10.4 +fresh-en-1,2,3,K,16,full_attention,200,0,200,1.0109024198900443,1.2307273106416687,1.609445944591425,1.3076426881743828,200,0.25361414849758146,0.31249481439590454,0.36692607998847954,0.3102434882521629,200,6.998705849878572,8.349572622676755,10.365628707007733,8.576305709125535,200,0.43741911561741076,0.5218482889172972,0.6478517941879833,0.5360191068203459,200,0.03623825147277862,0.061085228602307924,0.08576149902501261,0.060964982350360096,200,0.19415825673965206,0.24470573276556162,0.2871765057209829,0.24090295050963667,200,0.47765319903405207,0.5681173459011286,0.661245240865997,0.5658402826014348,200,0.7579121530518969,0.8237095466228523,0.9002946162473934,0.8237018166672861,200,0.013345774344385795,0.04020804644596182,0.07181708669559908,0.04387554096067126,200,8.0,11.0,12.0,10.34 +fresh-en-1,2,3,V,16,full_attention,200,0,200,1.174977968283929,1.4640721594914794,2.2168957058340313,1.5379768557913485,200,0.28450166285038,0.3321179151535034,0.4153254747390744,0.3424178659915924,200,7.203112711988059,8.82285433092217,10.558721022460047,8.896391024601092,200,0.45019454449925367,0.5514283956826356,0.6599200639037529,0.5560244390375683,200,0.034359517538111096,0.05423463634129673,0.081434205057719,0.056334825370769795,200,0.1801661729241178,0.2302969127972347,0.27514746940500384,0.2285761381612623,200,0.4846144489350299,0.5512079487733337,0.6434725919014037,0.5585333361548663,200,0.7366312354020832,0.8008326702210231,0.884389902679427,0.8032060818073257,200,0.015420266335058818,0.05351963470302416,0.08885071719981696,0.05408025950667003,200,9.0,11.0,12.0,10.86 +fresh-en-1,2,3,QKV,16,full_attention,200,0,200,1.1010740990925114,1.3543728347722208,1.8310944105382077,1.4172562011762784,200,0.2824027270078659,0.3214397579431534,0.3999712496995926,0.33167507566511634,200,7.051466896914471,8.452564179047128,10.582944184630527,8.717685363378575,200,0.44071668105715445,0.5282852611904455,0.661434011539408,0.5448553352111609,200,0.034124609663255125,0.059527793142735264,0.08460267987083449,0.059096589201155805,200,0.19049060067661536,0.24169465643832275,0.28761437780167387,0.23929467458421333,200,0.4733027615244746,0.5589565563114933,0.6517027008846887,0.5584453388754818,200,0.7445807477412757,0.8094723748407489,0.8941137978363501,0.8111042453298671,200,0.014149877714614578,0.04733354365090493,0.08122643701197324,0.04911841556807503,200,9.0,11.0,12.0,10.68 +fresh-en-1,2,3,JS,16,full_attention,200,0,200,0.03657998311018673,0.06375922867209738,0.13071855896851048,0.07590858518343452,200,0.017119598388671876,0.03612196817994118,0.088840002566576,0.04750879812519997,200,1.3517448154556186,2.7036518264869738,4.922167708676823,2.9735965507494266,200,0.08448405096597617,0.16897823915543586,0.30763548179230144,0.18584978442183916,200,0.1500401072024709,0.3278649442112621,0.7224388057118413,0.3764720703865471,200,0.37722934025425736,0.584493422319244,0.8574665387411939,0.5955056245592193,200,0.720753322212144,0.8402769636062397,0.9366637648614378,0.8338450409046326,200,0.8986704348182338,0.9500650508651199,0.9800354476704709,0.943634389750679,200,0.004016308725895021,0.01166079986047167,0.023390373518317843,0.013164300579311177,200,3.0,6.0,9.0,5.92 +fresh-en-1,2,3,sqrtJS,16,full_attention,200,0,200,0.5252659021643922,0.7200451572425663,0.9775034859310835,0.7384794788464205,200,0.1308414176106453,0.19005780667066574,0.29805870950222013,0.20634597267955543,200,4.794488709749097,8.205655685427681,10.553169124508095,8.009031013569468,200,0.29965554435931857,0.5128534803392301,0.6595730702817559,0.5005644383480917,200,0.03440937443169004,0.06332576025209877,0.15581127173801881,0.0780638007280054,200,0.1883847248399948,0.2682357262649282,0.42470076515085625,0.28363085772702634,200,0.49196037242739715,0.5682752056098923,0.6521272322138194,0.5669872715332271,200,0.7415683566855604,0.7903184435067119,0.8402944752219776,0.7894673620889282,200,0.05013157079692451,0.0726078709155922,0.095399863961596,0.0727427586290444,200,10.0,12.0,12.0,11.435 +fresh-en-1,3,4,Q,8,linear_attention,400,0,400,0.6302353186532855,1.1633159555494785,2.6859279543161394,1.4234560057288035,400,0.13086697906255723,0.2523481994867325,0.9357782542705538,0.3734714508987963,400,4.383198816307348,6.696189453983499,7.283896320271379,6.401085543894203,400,0.5478998520384185,0.8370236817479374,0.9104870400339223,0.8001356929867753,400,0.01404475587313086,0.02781565428533014,0.11788175231154865,0.042358775790364754,400,0.18575378664789088,0.22255037191057692,0.33922301411408406,0.2421743045740307,400,0.3524755893172402,0.40569014759217903,0.6114095392857971,0.4310910646621672,400,0.6207783981983105,0.6765808068366589,0.8192517033580835,0.6908469294748969,400,0.06848843453388288,0.1273126600131304,0.16097731206917185,0.12399106304632061,400,6.0,7.0,7.0,6.755 +fresh-en-1,3,4,K,8,linear_attention,400,0,400,0.7813272688537837,1.111222743988037,1.8391675867140296,1.214278789209202,400,0.15109480023384095,0.24492792785167694,0.43217862844467175,0.27287121130153535,400,5.987418858461102,6.911509401153365,7.364946802214619,6.7625808079952385,400,0.7484273573076378,0.8639386751441707,0.9206183502768274,0.8453226009994048,400,0.012318067921093941,0.02249851833851752,0.048020415420338275,0.02805273039436909,400,0.1780769246386972,0.20959856577888136,0.2749948617242884,0.2205960331691984,400,0.34033086231471443,0.38414510127490453,0.47210531273047385,0.39728468651716853,400,0.6191136328757632,0.6645397695013465,0.7263376296229,0.6675540183860753,400,0.10104444378861584,0.13444139693809745,0.1631149293432282,0.13336421373680626,400,7.0,7.0,7.0,6.9075 +fresh-en-1,3,4,V,8,linear_attention,400,0,400,0.6016671929508448,1.0942686591297388,2.0394407607615,1.2276800348144024,400,0.14361953884363174,0.27152132987976074,0.5094483792781831,0.30339992376044395,400,5.5165883677438625,6.316489463861998,7.129061453791907,6.317421319928843,400,0.6895735459679828,0.7895611829827498,0.8911326817239884,0.7896776649911054,400,0.017452482404810076,0.03807533032707222,0.0643102365004019,0.039890088080244156,400,0.19088940744593189,0.24409814792243176,0.3092172537329375,0.24692371281523873,400,0.36263943305493457,0.43297148796065926,0.5058827611379061,0.43460504711625303,400,0.6379878588705928,0.7003844085412088,0.7559172749534665,0.6985543461258836,400,0.08894995346497382,0.11722317104633828,0.14874281474397755,0.11774612696945606,400,6.0,7.0,7.0,6.7775 +fresh-en-1,3,4,QKV,8,linear_attention,400,0,400,0.8022432696074248,1.233229048550129,2.08050045222044,1.3658459807001053,400,0.1619195520877838,0.2708732634782791,0.6182220220565796,0.3241461719945073,400,5.016798247960995,6.819850935603542,7.276329876831179,6.595217795095848,400,0.6270997809951244,0.8524813669504427,0.9095412346038974,0.824402224386981,400,0.014207959176424883,0.024720881761060576,0.08494898609970297,0.033888299236001646,400,0.1819836350199073,0.2153251911428649,0.30612910375822694,0.22891742313977897,400,0.34864937689559816,0.39507204655668515,0.5526575667900605,0.4145404676041477,400,0.6233396079460133,0.6694136573472989,0.7732361047681235,0.678222176825968,400,0.08875942024456161,0.1327630917024994,0.15817161280058456,0.13001912991476752,400,6.0,7.0,7.0,6.86 +fresh-en-1,4,5,Q,16,linear_attention,400,0,400,0.5661739772418514,1.033524300903082,1.548737435415387,1.0487494651833549,400,0.07677488327026367,0.155758835375309,0.28011648654937743,0.16980655890889465,400,9.54058196238418,11.563004274180736,13.034847721078421,11.449623798232153,400,0.5962863726490113,0.722687767136296,0.8146779825674013,0.7156014873895096,400,0.015165402251029645,0.02558156261450213,0.04513649543366974,0.028092598404531807,400,0.11772207118899897,0.15330654570551983,0.21473730098518573,0.15949296593598414,400,0.408405519488417,0.4672246275226445,0.5331915149756827,0.468262158216055,400,0.6815111071027325,0.7377848861387815,0.7908779380323566,0.7359790336775207,400,0.05668044193959548,0.08783713953036076,0.11980524321628522,0.08880168565433808,400,11.0,12.0,13.0,12.1025 +fresh-en-1,4,5,K,16,linear_attention,400,0,400,0.6336847476661206,0.9515199940651655,1.3863349615596234,0.9780880847004301,400,0.08260805755853654,0.13552476465702057,0.23456682860851288,0.14650706651620568,400,10.439608141500683,12.080067106841595,13.448528931885292,12.001902712517099,400,0.6524755088437927,0.7550041941775997,0.8405330582428308,0.7501189195323187,400,0.012648100339200219,0.021633068318002303,0.035508306202475026,0.02332027475352894,400,0.11447453838353698,0.1405098994254031,0.1929915829477523,0.1483610316117159,400,0.3903108699302229,0.44735735197598236,0.500028661529038,0.44912106278242936,400,0.6736427389622814,0.720935236380688,0.7701319886219794,0.7191552692253345,400,0.06575611924543241,0.09578784114035244,0.13047202911319872,0.09687318871048539,400,11.0,12.0,13.0,12.3675 +fresh-en-1,4,5,V,16,linear_attention,400,0,400,0.3309630745323375,0.7136850020033307,1.6051203388720752,0.8373229334133794,400,0.05630024708807469,0.11041045933961868,0.2880571067333222,0.14496010718867183,400,8.94148971568207,10.822653477115722,12.748692361240215,10.833477778076572,400,0.5588431072301294,0.6764158423197326,0.7967932725775134,0.6770923611297858,400,0.017002045119772557,0.03189209693025585,0.0526275707115471,0.03383457575533693,400,0.12740913570226398,0.16342687899664543,0.22097331528843098,0.17146103671826857,400,0.42262395318296647,0.49338072208399364,0.5586564342871563,0.4936577956625194,400,0.6948036998118188,0.7575450949049569,0.8141788763919582,0.7556197234006445,400,0.04588552139353022,0.07646246560030266,0.11457910754686676,0.0789859277402695,400,11.0,12.0,13.0,11.7825 +fresh-en-1,4,5,QKV,16,linear_attention,400,0,400,0.6764165816130117,0.9491521252784878,1.4874159729108216,1.019960219575587,400,0.09058723673224449,0.13297109305858612,0.2412927031517029,0.15182253638282417,400,10.551028192725887,12.171941840187177,13.521055583087918,12.079058229642094,400,0.659439262045368,0.7607463650116986,0.8450659739429949,0.7549411393526309,400,0.012222641676116286,0.020966629665107612,0.03442934124906131,0.022584076082535383,400,0.11438074086551075,0.1415356176230052,0.18719380459025742,0.14692944830759347,400,0.3942103796029329,0.443952376087206,0.4990856825982414,0.4464748110851218,400,0.668005795100958,0.7149668152520021,0.7689533198016711,0.7164119723581883,400,0.06517239428564398,0.09687386971886072,0.13574425639454216,0.09878377784354246,400,12.0,12.0,13.0,12.42 +fresh-en-1,5,6,Q,16,linear_attention,400,0,400,0.7682637390680611,1.1547599723562598,1.569871773198247,1.1581397689768347,400,0.10521993413567543,0.17117173224687576,0.27427761256694794,0.18747375680133702,400,9.50916937846209,11.99832555853395,13.444038522158142,11.70992515950617,400,0.5943230861538806,0.7498953474083718,0.8402524076348838,0.7318703224691356,400,0.012674576634574011,0.022234630915170077,0.045505796523784385,0.026324687411875852,400,0.1166679813585501,0.149743468290625,0.21949883914153398,0.16130263591572055,400,0.39733354404507426,0.45121762880172855,0.5384325424157789,0.46037287157335305,400,0.6687125145085518,0.7184613386296534,0.7741422478278941,0.7184242785026582,400,0.07837240424458317,0.10372847255576656,0.13485728812541103,0.1057338280285208,400,12.0,13.0,13.0,12.6125 +fresh-en-1,5,6,K,16,linear_attention,400,0,400,0.8824158693663776,1.138687500730157,1.5312056209892035,1.172651090574218,400,0.10529714152216911,0.1579534038901329,0.24859169721603397,0.16948231166228653,400,11.122374400302377,12.824449968601705,14.054518441090336,12.666615471031482,400,0.6951484000188985,0.8015281230376066,0.878407402568146,0.7916634669394677,400,0.009228261448636185,0.016507814319700456,0.029236134897620388,0.018388176213125957,400,0.10866851795495212,0.13755310635705964,0.18935214486576998,0.143281900276671,400,0.37587373188771767,0.4245455589083859,0.4819297733510702,0.4279163926056065,400,0.6404680180888239,0.683179887961449,0.7342649586484324,0.6855171244908527,400,0.0954249142915524,0.12478354158782304,0.1521095252994814,0.1234154848653544,400,12.0,13.0,14.0,13.0375 +fresh-en-1,5,6,V,16,linear_attention,400,0,400,0.600328029692173,1.060917580500245,1.6198514576070016,1.088670219099149,400,0.09182117953896522,0.16631858050823212,0.27578833103179934,0.1759972857683897,400,9.629064485655736,11.646865993303294,13.3881649781128,11.580309188308082,400,0.6018165303534835,0.7279291245814559,0.83676031113205,0.7237693242692551,400,0.01300569703004901,0.024917341397089506,0.04410910254951378,0.027063130896668862,400,0.12008248659517151,0.15588521391184174,0.217150469704818,0.1630437327288194,400,0.3991636528156446,0.46413214973339423,0.5363752981769776,0.4650200921540183,400,0.664589266033564,0.721545676148293,0.7821924083983142,0.7238605905298994,400,0.06902164006677221,0.10247854526412727,0.13444669343255874,0.10158407651603575,400,11.0,13.0,13.0,12.4675 +fresh-en-1,5,6,QKV,16,linear_attention,400,0,400,0.9287251191213727,1.1830257810652256,1.4929384909570218,1.2071300790470558,400,0.11431095153093337,0.16038694232702255,0.2387257888913155,0.16842043263837694,400,11.309050699988678,12.849975342894167,14.050221972783795,12.77018758581391,400,0.7068156687492924,0.8031234589308854,0.8781388732989872,0.7981367241133693,400,0.009251470667743419,0.01634262982584435,0.027653324228697925,0.017498833204038878,400,0.11030839076921763,0.13414671001229264,0.17469306928174652,0.13900995628154095,400,0.3764104646535905,0.42312704727906564,0.4798279711508895,0.4262346805018312,400,0.6423216705584784,0.6838426029729531,0.725879868362153,0.685144730684903,400,0.09568884194494649,0.12515312729173686,0.15119528742610672,0.1240399041231213,400,12.0,13.0,14.0,13.0675 +fresh-en-1,6,7,Q,16,full_attention,200,0,200,0.704794529848732,0.8954973140498623,1.1695378178730607,0.9208162792466464,200,0.1738428220152855,0.21330231428146362,0.2548262745141983,0.21514354325830937,200,6.498744607130998,9.073556721814573,10.903215541274744,8.942558875538005,200,0.4061715379456874,0.5670972951134108,0.6814509713296715,0.5589099297211253,200,0.031163800902270482,0.05089107225524822,0.09746805860788076,0.05720957485443558,200,0.17822316353109774,0.22970062248760031,0.33465025538902443,0.24178357527688368,200,0.4745447265307109,0.5376676707461168,0.6040678766489488,0.5395352800605019,200,0.7439194321420506,0.7956452743552107,0.8331478132638561,0.7911566930851651,200,0.03624822054227332,0.058557099771943,0.08073051638287695,0.059066670396006174,200,10.0,11.0,12.0,11.125 +fresh-en-1,6,7,K,16,full_attention,200,0,200,0.7940170950023457,0.9337106640450656,1.2338959886692464,0.9765308360313065,200,0.16390912234783173,0.2076759785413742,0.25558628737926475,0.21144505679607392,200,8.235824914797698,9.377219017179222,10.874328966367425,9.493418418809252,200,0.5147390571748561,0.5860761885737014,0.6796455603979641,0.5933386511755783,200,0.03143234130466939,0.04708427445851279,0.06285562141828394,0.047040718485792936,200,0.17367197794562914,0.2193201045941068,0.26033378654654454,0.2185938032618412,200,0.47819825841861346,0.5299078290423314,0.5706757414676599,0.5284649050753172,200,0.7429876544186129,0.7886125552801035,0.8182859318605756,0.7838504224339157,200,0.039081856675619774,0.0603137502919301,0.08200966974593603,0.06214234327333388,200,10.0,11.0,12.0,11.24 +fresh-en-1,6,7,V,16,full_attention,200,0,200,0.9884055231232197,1.1532038510777056,1.3873046927154062,1.1716601063753478,200,0.1760020047426224,0.22081384807825089,0.2761884540319442,0.2278510296344757,200,8.26709590001093,9.970156464183948,11.709943200354465,10.090112921552624,200,0.5166934937506831,0.6231347790114967,0.731871450022154,0.630632057597039,200,0.024431228269953677,0.040319479365850175,0.06235926497435392,0.04108991007612572,200,0.14232486226591323,0.19181935540421316,0.2708529064044339,0.198712324921032,200,0.4589254331538943,0.5119421077228734,0.5646551666984734,0.508416543857184,200,0.7323016078995594,0.7702791821645893,0.8084478140089761,0.7708692957041913,200,0.04484347925458758,0.06700373403981082,0.0896610020970473,0.06768758477225605,200,11.0,11.0,12.0,11.48 +fresh-en-1,6,7,QKV,16,full_attention,200,0,200,0.8712185150827281,1.0402824106859043,1.223043517023325,1.0408493226702558,200,0.1742448091506958,0.2156040444970131,0.2560519903898239,0.21798965618014335,200,8.02160313162796,9.688187902457031,11.41896365440976,9.741112446786914,200,0.5013501957267475,0.6055117439035644,0.71368522840061,0.6088195279241821,200,0.0267452252926647,0.043433516698653155,0.06630807930044977,0.04465950294188996,200,0.1670947263614369,0.20846820360177704,0.2756778051675545,0.21290386664722902,200,0.46160249039682216,0.5224348477986831,0.5621384075676695,0.5168756794290319,200,0.7373330764017009,0.7815248457473716,0.8182293748928509,0.7767400682036921,200,0.04082117817627321,0.06183648480512356,0.08624314900080425,0.06445718695172137,200,11.0,11.0,12.0,11.425 +fresh-en-1,6,7,JS,16,full_attention,200,0,200,0.0210299493551247,0.041246538941777544,0.06721432701724551,0.043585980836786435,200,0.006701526325196028,0.01422842824831605,0.04112701267004013,0.01909851892036386,200,1.7860569878673398,4.5360171233995725,7.3734922631087505,4.67952068477081,200,0.11162856174170874,0.2835010702124733,0.4608432664442969,0.2924700427981756,200,0.07799573239916363,0.1684882364505519,0.5305697372661342,0.23399521490718442,200,0.2163755568674938,0.3725131729390068,0.7405501321902432,0.41946526146364504,200,0.6392109590204115,0.7961574337460801,0.9319402144416754,0.7893348187273534,200,0.8923583286138922,0.9466221129758281,0.9795742236133427,0.9393528461758904,200,0.00338059084708163,0.009482982066217775,0.022206033332710727,0.01184926370899521,200,3.0,6.5,9.0,6.34 +fresh-en-1,6,7,sqrtJS,16,full_attention,200,0,200,0.42382513666525484,0.6078069985378534,0.7762976725120098,0.6047175329021411,200,0.08186280280351639,0.11928264796733856,0.20279795229434966,0.12993449360132217,200,5.2684249272688914,9.284846390697087,11.584071745937548,9.020189238112819,200,0.3292765579543057,0.5803028994185679,0.7240044841210967,0.5637618273820512,200,0.025414074667080077,0.04821633030923124,0.13579863895911298,0.06206218609821719,200,0.13628849605637222,0.19926919792139358,0.3573190311116594,0.22364927416547772,200,0.46442194635302225,0.5545787239127964,0.6535459545865228,0.5606615276495901,200,0.7472960366904517,0.8005331227891845,0.8561625252391857,0.8010191938059725,200,0.04311398972753782,0.06126412435345474,0.08558115694072826,0.06341542019751516,200,10.0,11.0,12.0,11.1 +fresh-en-1,7,8,Q,8,linear_attention,400,0,400,0.9464878834784031,1.5582775697112083,2.9107058852911,1.7703495920123533,400,0.22571634501218796,0.36515119671821594,0.7337479174137118,0.418825199007988,400,5.637848997727274,6.390615838899661,6.92381149587291,6.322460295813479,400,0.7047311247159093,0.7988269798624577,0.8654764369841137,0.7903075369766849,400,0.022204722105925896,0.03597651159057591,0.05985445470533928,0.039036237734593235,400,0.19729815342566234,0.23558746903598987,0.28426601630860715,0.23892439141863886,400,0.3744167072230134,0.42127290344102886,0.48375565712163293,0.42576413196341584,400,0.6600986840065891,0.7129747388496561,0.7755491152602882,0.7161451590726728,400,0.08717807224644548,0.10753361942941522,0.13259106962929343,0.10864366387310449,400,6.0,7.0,7.0,6.6575 +fresh-en-1,7,8,K,8,linear_attention,400,0,400,0.29587780246511103,1.3316857907921076,2.1302281618118286,1.3505793981434544,400,0.06175224781036377,0.30029280483722687,0.5300436854362488,0.31198056460823864,400,5.730428692246048,6.436111467312488,7.008285784363738,6.400021282051126,400,0.716303586530756,0.804513933414061,0.8760357230454673,0.8000026602563908,400,0.020215161731297174,0.03471236497481557,0.05657945483029005,0.0369102295558765,400,0.19091308205700228,0.22275899268491542,0.27423060839622876,0.2293708406618569,400,0.36734390377955545,0.4109047477471825,0.483761550443415,0.41863879019956357,400,0.6574117720173502,0.713938634182789,0.766951627194338,0.7145666657366189,400,0.08712661344887594,0.11034598265737908,0.13472906965350992,0.1108709544837883,400,6.0,7.0,7.0,6.74 +fresh-en-1,7,8,V,8,linear_attention,400,0,400,0.6123314969241619,1.2732290644198656,1.9485007584095002,1.277252502203919,400,0.1477032333612442,0.3260002136230469,0.6659458816051484,0.3689683119766414,400,4.483468942423248,5.7863149132204175,6.767493150175568,5.710281588149037,400,0.560433617802906,0.7232893641525522,0.845936643771946,0.7137851985186296,400,0.02601741806177653,0.05465330811455165,0.11204833987527756,0.06285332811397336,400,0.20827549933278786,0.2612300592887439,0.38986242725355935,0.2809431239976787,400,0.38692017375926524,0.4609215685023522,0.5735366330265104,0.471533302387452,400,0.6731214069425167,0.7510156002949998,0.8286852509543848,0.7520838965871497,400,0.06267121005276431,0.09067035172824107,0.12475182415814548,0.09266975740687243,400,6.0,6.0,7.0,6.325 +fresh-en-1,7,8,QKV,8,linear_attention,400,0,400,0.931118180602789,1.4962615873664618,2.256703768670559,1.562762504387647,400,0.20242130309343337,0.35793085396289825,0.5517511188983918,0.368903941065073,400,5.624799861092076,6.332871957031749,6.894324056292317,6.293089391789616,400,0.7030999826365095,0.7916089946289686,0.8617905070365396,0.786636173973702,400,0.022910690305291522,0.03760713607041573,0.06032469450155453,0.039984388646460235,400,0.19804511860639032,0.23175319177899278,0.2953012330958894,0.23884446010755894,400,0.37836601710143425,0.42121551022909387,0.48087134422572936,0.4271086421246703,400,0.6706226669881539,0.7251146710314096,0.7764089088582178,0.7229980789228566,400,0.0895136743584083,0.10782311907684719,0.13184682203706802,0.10924205563047348,400,6.0,7.0,7.0,6.665 +fresh-en-1,8,9,Q,16,linear_attention,400,0,400,0.6070104320533574,0.9067567926831543,1.1999021530151368,0.9119760517461691,400,0.0825454406440258,0.1260710507631302,0.20973798036575317,0.13822114740498365,400,10.265056125706053,12.424898664612584,13.659821630652354,12.163357589789303,400,0.6415660078566283,0.7765561665382865,0.8537388519157721,0.7602098493618314,400,0.011421224708755817,0.019182463004563603,0.037246654788831664,0.022407370325958055,400,0.11157062818954842,0.14051746921552533,0.2027273658479494,0.14962862307951247,400,0.3890636778001099,0.4383249794075981,0.5092158490411423,0.44414624128338837,400,0.6631284831697636,0.7078375137427001,0.7588247426215273,0.7091028344739562,400,0.08445549662116993,0.10884091110247933,0.13445706357614887,0.10915460308997577,400,12.0,13.0,13.0,12.6875 +fresh-en-1,8,9,K,16,linear_attention,400,0,400,0.6323070602491498,0.8548832177184522,1.1772200994193556,0.8896699144935701,400,0.08282828703522682,0.11520854011178017,0.1833632454276085,0.12645583356730639,400,11.04570487773524,12.65198113751559,13.814199289209286,12.535487732930596,400,0.6903565548584525,0.7907488210947243,0.8633874555755804,0.7834679833081623,400,0.010548571932231885,0.017641606048966488,0.02990184148026892,0.019073353740446174,400,0.11078671461901378,0.1369087925384081,0.177843636786369,0.14159715183787902,400,0.38649372059484693,0.43143297944324893,0.49030923674098414,0.4358539541889845,400,0.6573873705846368,0.6965255522635809,0.7404600071183093,0.6971139947479787,400,0.09289461809305943,0.11826204424381498,0.14291525195667093,0.11880124152779212,400,12.0,13.0,14.0,12.93 +fresh-en-1,8,9,V,16,linear_attention,400,0,400,0.42908118362538517,0.7514993599615991,1.2147094633430242,0.7961950068455189,400,0.06230545081198215,0.10426175594329834,0.18898674845695496,0.11917674789205193,400,10.387996967089066,12.411267809027702,13.683809156777368,12.170210651725757,400,0.6492498104430666,0.7757042380642314,0.8552380722985855,0.7606381657328598,400,0.011284341988446791,0.0192767535918396,0.03601614703510141,0.022253234644721953,400,0.11100980234413184,0.14039500685822304,0.20002895542572496,0.14959486888672213,400,0.3848289486427891,0.43914656583209405,0.5086486354833948,0.44377163403348624,400,0.6610199499477317,0.705567551389675,0.7567553306241096,0.7075256323645357,400,0.08104022718925673,0.10993018657593058,0.13374696265975647,0.10907694315376375,400,12.0,13.0,13.0,12.695 +fresh-en-1,8,9,QKV,16,linear_attention,400,0,400,0.6932302778586745,0.8939938908442855,1.1676303837448359,0.9143451267119963,400,0.0818116083741188,0.11631309986114502,0.17395793199539186,0.12413974925875663,400,11.576339528650626,13.106297742226111,14.112753705414308,12.895273568385026,400,0.7235212205406641,0.819143608889132,0.8820471065883942,0.8059545980240641,400,0.00891508754003735,0.01471914409182325,0.025475298029896367,0.016848891123169132,400,0.1045271453356545,0.1291183437967932,0.18017723984847347,0.1360300248562872,400,0.37327479251058987,0.4161979816990303,0.47694368529170406,0.42015087939618573,400,0.6433031231226647,0.681086454830793,0.728392240051663,0.6841948586684446,400,0.09994393934612235,0.12528678116083536,0.14864559017689513,0.12464203221973566,400,12.900000000000006,13.0,14.0,13.07 +fresh-en-1,9,10,Q,16,linear_attention,400,0,400,0.5911897634156048,0.971417797729373,1.4385213758796456,0.9927398923510918,400,0.07862688973546028,0.14568262547254562,0.258398848772049,0.1655398803856224,400,8.301619620302397,12.424129421285567,14.065394468918116,11.809712500379186,400,0.5188512262688998,0.7765080888303479,0.8790871543073823,0.7381070312736991,400,0.009169578151820953,0.019187780270141216,0.06182346929316535,0.029226845717694877,400,0.1099473669490704,0.14248138560506401,0.2640789801094023,0.16535261853877542,400,0.3751454537343328,0.4365722274271774,0.5615250689219099,0.45259452274673345,400,0.6450473787167641,0.7024583459737376,0.7900427626121447,0.7115994346168725,400,0.06961536755566518,0.11083896103405731,0.1438247252029882,0.10823463941989513,400,12.0,13.0,13.0,12.6125 +fresh-en-1,9,10,K,16,linear_attention,400,0,400,0.5946242531761528,0.8478443613275886,1.1840944074094295,0.871135561830015,400,0.07446554973721504,0.11519348248839378,0.16995172798633576,0.12028403769247234,400,10.549285512368025,12.972315807446954,14.425008287211448,12.712822515189973,400,0.6593303445230015,0.8107697379654346,0.9015630179507155,0.7945514071993733,400,0.00727898771604036,0.01555976747433167,0.03444604154887297,0.018506599106899665,400,0.10209242735246339,0.12992744304582096,0.19340092297569486,0.13964911344364841,400,0.3592895824498176,0.41231036761758694,0.5098439554283221,0.4227395498424912,400,0.6313341086885862,0.6780047290866777,0.7607935107506781,0.6888854020278815,400,0.0812711923511712,0.12499023766221395,0.14986821912439047,0.11892558068973458,400,12.0,13.0,14.0,12.885 +fresh-en-1,9,10,V,16,linear_attention,400,0,400,0.46655475879088043,0.694292152300477,1.1977318368852141,0.7830342625826597,400,0.05845900662243366,0.09787659719586372,0.18204079568386078,0.109708200301975,400,10.79670724624694,12.799561081491053,14.098469985754189,12.585144444076532,400,0.6747942028904338,0.7999725675931908,0.8811543741096368,0.7865715277547832,400,0.00899166930099007,0.016669541140850203,0.032128917791397246,0.01915084256542966,400,0.10634733888134494,0.13420035804145325,0.1814061567037344,0.14053209500203084,400,0.3705575396237194,0.4225866918800977,0.4920889905616666,0.4271593782406484,400,0.6446972305378428,0.6932596927957679,0.7563180219386684,0.6969348333017064,400,0.07840079338564739,0.11551890932463348,0.14459395356534224,0.11412406729654365,400,12.0,13.0,14.0,12.7825 +fresh-en-1,9,10,QKV,16,linear_attention,400,0,400,0.6377313435077667,0.9152962598018348,1.2620975852012637,0.9343612542306073,400,0.07677633911371232,0.11948640644550323,0.19208532124757768,0.13030948131345213,400,10.357105699830317,13.281428814238446,14.563325631726023,12.857813217521873,400,0.6473191062393948,0.8300893008899028,0.9102078519828765,0.803613326095117,400,0.0065766772212237975,0.013645997606642514,0.03632221423452987,0.018165359049485318,400,0.09717792129277648,0.12738703947564595,0.19875304423416926,0.1397535679882335,400,0.3500467030986209,0.40565446005452366,0.4903084005348662,0.41452393551571715,400,0.6224946321414305,0.6713920031240022,0.7545570327695006,0.680820957911161,400,0.083562055059871,0.12944488972430918,0.15645529725672752,0.1245111882230568,400,12.0,13.0,14.0,13.0275 +fresh-en-1,10,11,Q,16,full_attention,200,0,200,0.6233514178544283,0.8477539476007223,1.1313497941941022,0.864531800642144,200,0.09261990860104562,0.12799295783042908,0.16213646531105042,0.12703987561166286,200,11.083121410511584,12.33530870232173,13.516670229505364,12.303993275644102,200,0.692695088156974,0.7709567938951081,0.8447918893440852,0.7689995797277563,200,0.012248285507573719,0.019806210387773356,0.029575802469652645,0.020589419749336493,200,0.12059200021166196,0.14568953492165526,0.1819297448828569,0.14910866795322417,200,0.40007387310569625,0.43991574423397506,0.4870083295202077,0.44172911545795057,200,0.6633287162181966,0.702975283069781,0.7459089131971716,0.7032529186849978,200,0.08783156294418897,0.11597740218955471,0.13523742874716974,0.11330198812926309,200,12.0,13.0,13.0,12.82 +fresh-en-1,10,11,K,16,full_attention,200,0,200,0.6800970683805645,0.847108400426805,0.9841707344166932,0.842388323675841,200,0.0932523988187313,0.18622612953186035,0.2628218740224838,0.18055713117122651,200,6.831904644430433,10.135327225600486,12.63554945002832,10.042840061771924,200,0.4269940402769021,0.6334579516000304,0.78972184062677,0.6276775038607453,200,0.017751430611250053,0.038580493450923,0.08946885431236626,0.046074088112288365,200,0.12633302196797447,0.1970723329010813,0.32237500984223744,0.2120215437822686,200,0.4337426493856024,0.5097912751392226,0.5820843687969331,0.5114911555571688,200,0.7047975019694904,0.7459314267217192,0.8060394526965863,0.7514484662784647,200,0.06533122729366882,0.09748410784204035,0.12131077370282961,0.09361191047261495,200,11.0,12.0,13.0,12.22 +fresh-en-1,10,11,V,16,full_attention,200,0,200,0.7686313582584262,0.9857306042686105,1.1916024396196008,0.9675404566898942,200,0.0949378065764904,0.12796563655138016,0.21206880509853362,0.1381375689804554,200,11.073414657827414,12.676510619831506,13.880663018912692,12.610443800586177,200,0.6920884161142133,0.7922819137394691,0.8675414386820433,0.7881527375366361,200,0.010178904699318604,0.01747881414906011,0.029660328925843533,0.018491370356752187,200,0.10627799103028163,0.1370622484430256,0.19449539198847193,0.14159127914808728,200,0.38537925019478797,0.4288803086713212,0.4663899522615944,0.4283227395377166,200,0.6557914448707022,0.6953265317172332,0.7364893087064365,0.6973511955899485,200,0.09234897368639247,0.1197079753559683,0.1391725013906082,0.11682758756430534,200,12.0,13.0,14.0,12.9 +fresh-en-1,10,11,QKV,16,full_attention,200,0,200,0.7310712728649378,0.9247950157150626,1.100934425368905,0.9188851007819175,200,0.09555794224143029,0.14651821553707123,0.1867323637008667,0.14397828802466392,200,10.181497428447223,12.162679164102345,13.51513785670134,11.95559962381206,200,0.6363435892779514,0.7601674477563966,0.8446961160438338,0.7472249764882537,200,0.012257180879865775,0.021033367430259478,0.03809874098679807,0.023718019033761684,200,0.11219114064328034,0.1534239971476366,0.21174015876909552,0.15776689704574276,200,0.40151250960251883,0.44164458518768407,0.5117346580720709,0.45302702288091495,200,0.6732404801270179,0.7152158421781707,0.7554212286570804,0.7151006381798125,200,0.08130715972510741,0.1125337841224287,0.13469992492262145,0.10986820736768976,200,12.0,13.0,13.0,12.735 +fresh-en-1,10,11,JS,16,full_attention,200,0,200,0.012129754142551973,0.02756568178301677,0.053156213027978086,0.03111563401675994,200,0.0035617715446278455,0.008189627435058355,0.022515043430030347,0.012655710716499015,200,2.772891556438018,5.620292373824549,8.392975529674137,5.65763210199563,200,0.17330572227737612,0.3512682733640343,0.5245609706046336,0.3536020063747269,200,0.06042468716479553,0.12312224875165927,0.3180355820918079,0.17066951470487632,200,0.1990284378536723,0.3222974385676557,0.577510841922257,0.36134668632905187,200,0.5958839053122645,0.7174585738399295,0.8489595309124881,0.7218225477606007,200,0.8345675211164408,0.8958787219369339,0.953208964678737,0.8971598878108424,200,0.01245711316804196,0.028983649447609384,0.05255686712975372,0.03081421530304316,200,6.0,9.0,10.099999999999994,8.225 +fresh-en-1,10,11,sqrtJS,16,full_attention,200,0,200,0.3480386174749583,0.543599259108305,0.7279269386082887,0.5426706708426354,200,0.05968044437468052,0.09049655869603157,0.1500500649213791,0.10147279778495431,200,7.786795436381748,11.167258724565684,12.856941973850715,10.717228882353261,200,0.48667471477385926,0.6979536702853553,0.8035588733656697,0.6698268051470788,200,0.016297805213144903,0.028850671209787696,0.07032379809211387,0.03941906345986685,200,0.126183576611211,0.17118246445915597,0.27297499398166714,0.1883892132250232,200,0.4253922406250987,0.4836082610551373,0.5842930396744872,0.49745094528228123,200,0.6902138798479083,0.7384927740399874,0.8014818866030737,0.7431939922447773,200,0.07471577023129482,0.09971293189387676,0.128884513148028,0.09995610012552845,200,12.0,12.0,13.0,12.375 +fresh-en-1,11,12,Q,8,linear_attention,400,0,400,0.7567769389599562,1.0795705001801252,1.7580996453762059,1.1694565903116017,400,0.18596988320350646,0.29172611236572266,0.4819856643676758,0.31001477092504504,400,5.524607274825687,6.414253687847145,6.880443063052533,6.308297452353024,400,0.6905759093532109,0.8017817109808931,0.8600553828815666,0.788537181544128,400,0.023245134263822177,0.035317468085793564,0.06400953881333335,0.03964568935568683,400,0.21995468508733446,0.25814923850456784,0.31921119443047025,0.2647836258684623,400,0.3848805435459116,0.4195852463063549,0.4855266401589614,0.42801073949973056,400,0.6484030977828479,0.6830797912060882,0.7350299329888763,0.6863293791390171,400,0.09359781213000229,0.11960744618172511,0.1483041568860997,0.12006861918145079,400,6.0,7.0,7.0,6.8175 +fresh-en-1,11,12,K,8,linear_attention,400,0,400,0.5281048333272338,0.8310451991856098,1.4153220526874066,0.9152459127036855,400,0.12563560381531716,0.21257421374320984,0.3468866616487503,0.22543062875047326,400,5.941233116580824,6.53863616173657,6.964164763719883,6.492122490216555,400,0.742654139572603,0.8173295202170713,0.8705205954649854,0.8115153112770693,400,0.021248283466748612,0.03192810778996481,0.04950315486757437,0.033915335080159884,400,0.20772005638230942,0.24559798928419585,0.2896104520395671,0.24772033961933929,400,0.37449797872241064,0.4094081479182775,0.46352951685001714,0.4142904575526047,400,0.6394769970639034,0.6769673642323686,0.7263123591199059,0.6801635151190543,400,0.08979062402246012,0.1212878312681364,0.15436371857201703,0.12109071700773352,400,6.0,7.0,7.0,6.7875 +fresh-en-1,11,12,V,8,linear_attention,400,0,400,0.40966571010649205,0.7090068785473704,1.3721489839255814,0.822325454945676,400,0.0932987503707409,0.17045152187347412,0.3586284995079041,0.20707087761722506,400,5.70724240881158,6.530690915793778,7.17927972446588,6.47746376894724,400,0.7134053011014475,0.8163363644742222,0.897409965558235,0.809682971118405,400,0.01633113148272599,0.03214076854143387,0.05738976178152401,0.034936112599367214,400,0.19416638584896312,0.2435624165748207,0.30346263875665414,0.245877599984745,400,0.360049786245077,0.4113513954222471,0.47683463113878694,0.41534938462501414,400,0.6313436976228772,0.683580585877189,0.7369502103048826,0.6839693390150956,400,0.08468630236873252,0.12112264009059343,0.15558656302894555,0.12173724458077556,400,6.0,7.0,7.0,6.8 +fresh-en-1,11,12,QKV,8,linear_attention,400,0,400,0.6695722434669733,0.9737617634236813,1.4699242446571588,1.0259376918058842,400,0.15512494146823883,0.24131304770708084,0.37777670025825505,0.25822414975613356,400,5.917319637446901,6.573636620480408,6.968569728815417,6.508530547434167,400,0.7396649546808626,0.821704577560051,0.8710712161019272,0.8135663184292709,400,0.0211445407853598,0.03099748534940353,0.05028049688278485,0.03342526924951561,400,0.21369708065143506,0.24717857328140533,0.2961957323016306,0.250994432492419,400,0.3753533510823,0.4094599139837099,0.4594258168986935,0.4134366145626896,400,0.6394669697597971,0.6769448204384146,0.715005530353559,0.6761832911579259,400,0.09667641657455553,0.12475849116511666,0.1550902544220896,0.124751852060638,400,6.0,7.0,7.0,6.86 +fresh-en-1,12,13,Q,16,linear_attention,400,0,400,0.40015879357233647,0.6367415268905461,0.9498837446328253,0.6584214791984414,400,0.053000422194600105,0.10679681226611137,0.21886933296918876,0.12330687199253589,400,7.900701295613523,11.646988774273883,13.565833801249099,11.169689381342009,400,0.49379383097584517,0.7279367983921177,0.8478646125780687,0.6981055863338755,400,0.011962247618184348,0.024916378273358276,0.06834302681699897,0.033075099284526115,400,0.1179872683467821,0.16569685816194646,0.2649923981204602,0.17874599109264844,400,0.38778811481556,0.45604739048671195,0.5857442320003411,0.4715996333051211,400,0.6629149618346649,0.7209183367904728,0.7998450453295818,0.7251634450902051,400,0.06120596826559935,0.09968537422661808,0.1256721898950002,0.09558489290132244,400,11.0,12.0,13.0,12.325 +fresh-en-1,12,13,K,16,linear_attention,400,0,400,0.3927572287619114,0.5375448940321803,0.7774453839287162,0.565406767456443,400,0.049797373637557034,0.07518744841217995,0.1224941782653332,0.08126530165784061,400,10.335213078032655,12.68344582330926,14.048625770169288,12.365775688776461,400,0.645950817377041,0.7927153639568287,0.8780391106355805,0.7728609805485288,400,0.009260105482935049,0.01743246431054096,0.03654036661214121,0.021138451426572388,400,0.10327901904619319,0.1349392011451908,0.19822484513796557,0.14440075104755393,400,0.3664113750642849,0.423616482568947,0.5049829225435034,0.43129149945616496,400,0.643231395495744,0.6959097085110486,0.7615328337640832,0.6990269813214163,400,0.06919068340026685,0.11007263677564436,0.13499486824500956,0.10656551438191535,400,12.0,13.0,13.0,12.595 +fresh-en-1,12,13,V,16,linear_attention,400,0,400,0.3575849617598578,0.4940988947637379,0.6994051075074822,0.5102940507073072,400,0.04921595938503742,0.07432536780834198,0.12480693981051447,0.08175155418924987,400,9.18300042738304,11.822130690188416,13.535588381373007,11.550608502097361,400,0.57393752671144,0.738883168136776,0.8459742738358129,0.7219130313810851,400,0.012137938596258436,0.023559714933594798,0.04949017537220834,0.027914165765983254,400,0.11513807661584576,0.15112549001669207,0.224298238511075,0.16088247286924087,400,0.38371161510986945,0.4512482306128598,0.5533690730604433,0.46237825614832745,400,0.664523578389823,0.7248640452356558,0.795254827341673,0.7286026835665986,400,0.05957107863163338,0.09388339442183025,0.12476552595569422,0.09261185332096161,400,11.0,12.0,13.0,12.2025 +fresh-en-1,12,13,QKV,16,linear_attention,400,0,400,0.4490968918427825,0.6040551688056439,0.772013966459781,0.6097000751754967,400,0.056225146353244784,0.08773870766162872,0.14940151125192652,0.09616111494600772,400,9.310497508280006,12.463384370486093,14.001834520812338,12.040878292479833,400,0.5819060942675004,0.7789615231553808,0.8751146575507711,0.7525548932799896,400,0.009513837982407871,0.018917363782731116,0.047899355805696736,0.024104700080375643,400,0.1075846235782156,0.14413174851535884,0.22382376080225969,0.1559608068805977,400,0.3710945911912024,0.43201796985240454,0.5350885532393697,0.44223589811974845,400,0.6432294246449133,0.6999073139039137,0.7763119762949903,0.7049439654665777,400,0.06982721825175492,0.11029844876864406,0.13662473491330956,0.10616657765486029,400,12.0,13.0,13.0,12.6025 +fresh-en-1,13,14,Q,16,linear_attention,400,0,400,0.38587391609326005,0.6040984917199239,0.9251342911273243,0.6330764132403419,400,0.05596090294420719,0.10490240156650543,0.17073518931865694,0.10768222124082967,400,9.049924681683754,11.20894886640028,12.745485458953814,11.019996260893349,400,0.5656202926052346,0.7005593041500175,0.7965928411846134,0.6887497663058343,400,0.017023126484248807,0.028495393526881914,0.05119803810629733,0.03193805800542335,400,0.12355473699501456,0.16101374796579332,0.22482045349795793,0.16916563044889554,400,0.42063078432393225,0.48025343701919054,0.5569519138818774,0.4854138120642995,400,0.6984053024785097,0.7484979737849548,0.8014359438012052,0.7488587160328416,400,0.05877644099045833,0.08314924996652645,0.11196985507825112,0.08361870610330539,400,11.0,12.0,13.0,11.9275 +fresh-en-1,13,14,K,16,linear_attention,400,0,400,0.43110426678322256,0.6153574767522514,0.93125104540959,0.6522414822975406,400,0.06342344433069229,0.10198630392551422,0.17182498276233674,0.11229783963412046,400,9.466881356501704,11.26579073181909,12.882829360778084,11.182182592684812,400,0.5916800847813565,0.7041119207386931,0.8051768350486302,0.6988864120428008,400,0.01613104306962981,0.028015269875100807,0.04600683653386488,0.030472140818704318,400,0.12392575955873118,0.16394327476466236,0.2241900720502706,0.17025395484046738,400,0.4159190131955582,0.4775634959011829,0.5445528443290275,0.47872412709523054,400,0.6873734845036722,0.7383968589247769,0.7847045412168916,0.7388570560911569,400,0.06455250246653593,0.09181227828863649,0.11854492766760194,0.0919839941054243,400,11.0,12.0,13.0,12.195 +fresh-en-1,13,14,V,16,linear_attention,400,0,400,0.2900645933113992,0.48444348643533885,0.9060238415375352,0.5558347255448461,400,0.04291451759636402,0.08081001043319702,0.15883952975273133,0.0956442926172167,400,9.082088108694267,11.182737128421905,12.937344798062185,11.029474177962703,400,0.5676305067933917,0.6989210705263691,0.8085840498788865,0.6893421361226689,400,0.015782789673193617,0.02871844968737506,0.05078076652324791,0.03287630538483202,400,0.12097299468637882,0.16364300191529996,0.2316634520292726,0.17091962330260344,400,0.4107163934931436,0.4782539947641684,0.5566652830327425,0.4834321985556987,400,0.6858478089555013,0.7493497825824824,0.8054376502573247,0.7476701447073734,400,0.057861569823473076,0.0836187740495665,0.11742488317182441,0.08550577094016727,400,11.0,12.0,13.0,11.9875 +fresh-en-1,13,14,QKV,16,linear_attention,400,0,400,0.44669262883253397,0.623176051536575,0.895405951049179,0.6510813516378403,400,0.06379591152071953,0.09507973119616508,0.15422177463769912,0.10377238671295345,400,9.936992727120396,11.636051701966881,13.048391316169603,11.536166028030898,400,0.6210620454450247,0.7272532313729301,0.8155244572606002,0.7210103767519311,400,0.015080325965394503,0.02500251873711617,0.04067634708726738,0.027247416794218743,400,0.11910349186550723,0.15268726401867833,0.20756579424184052,0.15914044390480855,400,0.408817362329244,0.4631131582779432,0.526147748039087,0.4668917532067847,400,0.684571216698779,0.7332940035915071,0.7834012840377015,0.7323544734283491,400,0.06558309125821403,0.09340692378298066,0.11996441808490575,0.09349140361054459,400,11.0,12.0,13.0,12.25 +fresh-en-1,14,15,Q,16,full_attention,200,0,200,0.6563409255351871,0.8885973512660712,1.1189436026848854,0.8895391296368325,200,0.10744329616427421,0.13625537604093552,0.16880006939172745,0.13614295069128274,200,10.06622890678192,11.3995462231671,12.568105344479184,11.298629109686868,200,0.62913930667387,0.7124716389479437,0.785506584029949,0.7061643193554292,200,0.018204254305053534,0.026904311137326876,0.03929820876447321,0.02843344256938163,200,0.1273237626847258,0.15231394655018,0.18881087880584652,0.15567140948879182,200,0.43790628827696393,0.48271703602284305,0.5308263678853762,0.4824258262657118,200,0.7068053089766504,0.7451939087679385,0.7798337437196036,0.7443418419102259,200,0.0686915516833074,0.08643757504508862,0.10901966952622595,0.08786601411126468,200,11.0,12.0,13.0,12.09 +fresh-en-1,14,15,K,16,full_attention,200,0,200,0.4422917545773089,0.6692723147571087,0.8980086565949023,0.6666717913839966,200,0.08063044473528862,0.13187910616397858,0.26484550237655635,0.1580746714770794,200,6.496921061890318,8.634981013580475,10.647325722358193,8.592294287572662,200,0.4060575663681449,0.5396863133487797,0.6654578576473871,0.5370183929732913,200,0.03351509067240662,0.056861881560029734,0.09751663408847647,0.06317353693975691,200,0.16137887751080895,0.2089274880791548,0.307736514048632,0.22779521059396768,200,0.5046569457559306,0.5892182690968026,0.6774019030472233,0.5882521444102824,200,0.7691001509701026,0.8213856097899936,0.8655278039685198,0.8219825699711337,200,0.03543520661033324,0.0521927683576505,0.07109596605144249,0.05222577504585699,200,10.0,11.0,11.099999999999994,10.58 +fresh-en-1,14,15,V,16,full_attention,200,0,200,0.38464598446153103,0.5482084071263671,0.8379248509416356,0.5899795142468065,200,0.06866812258958817,0.11133556813001633,0.24260862022638316,0.13773065745830537,200,6.216467446575847,8.988623652207185,10.989150092716981,8.848462383194583,200,0.38852921541099045,0.561788978262949,0.6868218807948113,0.5530288989496615,200,0.030398839890936464,0.05200406482839079,0.10492072590861068,0.061358771557470666,200,0.16093596423522352,0.2045543545965087,0.3259517905663392,0.2280630588249819,200,0.49637822330059916,0.5724431643306074,0.6667465477694386,0.5754301473773614,200,0.7521417976747519,0.8015559040088436,0.842109261694103,0.7990237153134504,200,0.049456321819029994,0.06213600352255539,0.08528731593998064,0.06430578171829107,200,10.0,11.0,12.0,11.16 +fresh-en-1,14,15,QKV,16,full_attention,200,0,200,0.6223420369904489,0.778354502748698,0.9231582049280405,0.7789445933763637,200,0.10168107002973557,0.1296810358762741,0.18023506850004192,0.1366561456397176,200,8.41350180833179,10.523084858081432,11.822431940106984,10.315722761810889,200,0.5258438630207368,0.6576928036300895,0.7389019962566865,0.6447326726131806,200,0.02355731133370658,0.034697786419297105,0.060114480614131806,0.03902258723578814,200,0.1334137715490158,0.16518611109273196,0.2417386911606085,0.1778114040581989,200,0.4630055955557551,0.5157883506966777,0.589183515042034,0.5209355972429051,200,0.7326387455106063,0.7717549500481755,0.8128654685207433,0.7724085128803674,200,0.05658227244761688,0.07334248693672096,0.0939632807903904,0.07454463698491803,200,11.0,12.0,12.0,11.63 +fresh-en-1,14,15,JS,16,full_attention,200,0,200,0.018004731212204205,0.03348266697184954,0.059227829484098035,0.03692166017076658,200,0.004639329807832837,0.010504491161555052,0.025016705133020872,0.013612156253657304,200,3.274025923895891,5.125033934823258,7.78695510128457,5.334980079610843,200,0.20462662024349318,0.32031462092645363,0.4866846938302856,0.3334362549756777,200,0.07031469472045071,0.14146500516297272,0.25914426730502294,0.16129882478698312,200,0.2029433759326427,0.32494952021570706,0.5013753590550546,0.34711214847798033,200,0.6287480978828671,0.7532603672391712,0.8900012906240546,0.7571391532104915,200,0.8834822446646576,0.9343652837231533,0.9743636498315802,0.9299282811985253,200,0.004901058654277581,0.013966580533476772,0.028204854633257292,0.015356057073776968,200,5.0,7.0,9.0,7.035 +fresh-en-1,14,15,sqrtJS,16,full_attention,200,0,200,0.4415515260305256,0.5724745735060424,0.7562368151731789,0.5807246377062983,200,0.06811262890696526,0.10248782485723495,0.15816269665956495,0.10990720879286528,200,7.529556492098475,10.043100982924784,11.830367405808222,9.857777920876396,200,0.4705972807561547,0.627693811432799,0.7393979628630138,0.6161111200547748,200,0.02349704647211202,0.03954228448838887,0.07499737416111749,0.04573978544263805,200,0.13440912438352798,0.1823663584313931,0.2529708800712911,0.19014998749873235,200,0.4534672937478886,0.5284547764820187,0.6352246885760922,0.5384655484043196,200,0.737841351853865,0.7883864326779022,0.8530867643748598,0.789249829856754,200,0.046555005837892384,0.07025507592106536,0.09329942047575829,0.07066103067246497,200,10.0,11.0,12.0,11.305 +fresh-en-1,15,16,Q,8,linear_attention,400,0,400,0.2711366474628448,0.539630301296711,0.9237230136990549,0.573938003234798,400,0.06328783221542836,0.12822913378477097,0.2760877221822739,0.14860856218729168,400,5.576764567866008,6.681737031598869,7.362441183548381,6.579373347122547,400,0.697095570983251,0.8352171289498587,0.9203051479435477,0.8224216683903184,400,0.012370874578419209,0.028184786627722263,0.06207479661467802,0.0328838024600782,400,0.19149372017339686,0.24407549472571974,0.3228724132773278,0.25115620622294166,400,0.35086566362338967,0.4071006429210248,0.48109258782423536,0.41288698567939136,400,0.6123027580780799,0.6550810994024083,0.7184733863773753,0.6595501142579798,400,0.11872841197973748,0.1463155329486811,0.1737485704960282,0.14680050892723961,400,7.0,7.0,7.0,7.0025 +fresh-en-1,15,16,K,8,linear_attention,400,0,400,0.34259005822241306,0.5003784280270338,0.7856600016355515,0.5381201247498393,400,0.06735700890421867,0.11001741141080856,0.19674786627292637,0.12210885928012431,400,6.251888971033041,7.025771520366799,7.396969407384826,6.892041953274288,400,0.7814861213791301,0.8782214400458499,0.9246211759231032,0.861505244159286,400,0.011646288671032264,0.01980928311826797,0.03994475122366749,0.023928021941082102,400,0.18092751893393544,0.21356752463073111,0.27390740614220743,0.22162753673329508,400,0.34312823181484314,0.38427251972638876,0.4490570748420421,0.3913343810625774,400,0.6107786398562652,0.6495456871563821,0.6937249247544216,0.6514891970104955,400,0.12386306464319126,0.14860336645650923,0.17426210665525194,0.14963481543350796,400,7.0,7.0,7.0,6.995 +fresh-en-1,15,16,V,8,linear_attention,400,0,400,0.32907226383686067,0.47515508625656366,0.727832093089819,0.5049836496147327,400,0.06769566759467124,0.1027485616505146,0.1874361485242844,0.11676882148720324,400,5.945369281835923,6.912787387200893,7.3764059772068675,6.765377967118086,400,0.7431711602294904,0.8640984234001117,0.9220507471508584,0.8456722458897608,400,0.012077009402636885,0.022467939320686185,0.04936930644935354,0.027807225289983296,400,0.1816270836641393,0.2133201308117083,0.2924820619928854,0.22703727848122437,400,0.34614924678364245,0.38837277611282234,0.47123153788227584,0.39795358557295385,400,0.6189158925670177,0.6598919231859717,0.7159850313211588,0.6623270262518922,400,0.11070638041674453,0.14096662480505417,0.16865267685249227,0.14042823882967237,400,7.0,7.0,7.0,6.96 +fresh-en-1,15,16,QKV,8,linear_attention,400,0,400,0.3923039147630334,0.5221432708203793,0.8025936536490917,0.5630558036244474,400,0.07713610976934433,0.11622476205229759,0.21189949959516538,0.13013154728338122,400,6.155570420277679,7.017005264912422,7.408260599098569,6.907467952094582,400,0.7694463025347099,0.8771256581140527,0.9260325748873212,0.8634334940118228,400,0.011410804437885837,0.020012503240417603,0.04280531834209489,0.023445391863559872,400,0.18253702129454166,0.22002798493699968,0.28081590109501725,0.2258137937300615,400,0.34338422188178963,0.38652627350977825,0.4507843386809182,0.39107108723694867,400,0.6111723327343769,0.642529598383496,0.6877122119536123,0.6455973608547378,400,0.13406645195545083,0.1538841811553101,0.1776088371211376,0.1546207257540409,400,7.0,7.0,7.0,7.005 +fresh-en-1,16,17,Q,16,linear_attention,400,0,400,0.09337918295641431,0.30953697045333683,0.5907246937509627,0.3352472458622651,400,0.01392392572015524,0.05418849736452103,0.10268680155277254,0.05841724322293885,400,8.905798445796412,11.64138146547818,12.921808616472704,11.213610030850402,400,0.5566124028622758,0.7275863415923862,0.807613038529544,0.7008506269281501,400,0.01588112491850402,0.024960488715059535,0.0531152409056388,0.03173127547388234,400,0.12080590748803613,0.15235131259761714,0.2515795113947713,0.17120790128583668,400,0.41127631446019447,0.4632548207555784,0.5408936029680277,0.4723083477384988,400,0.6856202056846542,0.7341247612521012,0.7819669842072424,0.7346803970212874,400,0.06677120987431369,0.09165393634143856,0.11694633328234391,0.09132242361102115,400,11.0,12.0,13.0,12.19 +fresh-en-1,16,17,K,16,linear_attention,400,0,400,0.3151325881015509,0.4342981765512377,0.6140281042549759,0.4490922313547344,400,0.043734886497259144,0.0605989508330822,0.0972646735608578,0.066486811270006,400,10.724829550181356,12.228698399775592,13.22123826722373,12.06350940830659,400,0.6703018468863348,0.7642936499859745,0.8263273917014832,0.7539693380191619,400,0.014011608106958827,0.020559842157920602,0.0327910838453225,0.022651020110824805,400,0.11532941394132296,0.14109018363332754,0.18747700691390254,0.14804679437938673,400,0.4007285213311116,0.4402400299003572,0.4938822597033423,0.4448768679799309,400,0.6807477272333515,0.7168661065576287,0.7564023618453122,0.7174010597407197,400,0.07520501474426387,0.09555361683928279,0.1174611252098421,0.09637732141903163,400,12.0,12.0,13.0,12.3675 +fresh-en-1,16,17,V,16,linear_attention,400,0,400,0.27350447163917124,0.4227070201886818,0.5940568962134422,0.4355280553940247,400,0.03790762796998024,0.0634639710187912,0.10251315906643868,0.06916116249281913,400,9.488318210046588,11.943810424407413,13.194843364123189,11.569463788469852,400,0.5930198881279117,0.7464881515254633,0.8246777102576993,0.7230914867793657,400,0.014172994549902618,0.022640399642703552,0.045752280059905594,0.027438945403027885,400,0.11746019499638564,0.1448118464364499,0.21618551729597588,0.15761362888869188,400,0.4033386966923207,0.45345510537966793,0.5382468362442999,0.46324582845119655,400,0.6809839603722169,0.7250734270160011,0.7918636445196023,0.7326652321697534,400,0.05918017447726996,0.09173037870574513,0.11421132244248275,0.08876760141047736,400,11.0,12.0,13.0,12.07 +fresh-en-1,16,17,QKV,16,linear_attention,400,0,400,0.3224496560636908,0.42696445202454925,0.5670095440000296,0.43686625730391826,400,0.04111108183860779,0.05806395970284939,0.0970219224691391,0.06411537552252412,400,10.860585927689414,12.332699332159685,13.303101069674536,12.185908437883166,400,0.6787866204805884,0.7707937082599803,0.8314438168546585,0.7616192773676979,400,0.013515142884425336,0.019824265945808238,0.03154781249084804,0.021555262088733596,400,0.11412806625684199,0.13841783456198153,0.18926734850591428,0.14527515172162736,400,0.3977816898172273,0.43762988539490966,0.4904281217063621,0.4405206492400464,400,0.6755264447647816,0.7127760679403246,0.7514245675042959,0.7138758245488483,400,0.07636906628376562,0.09805705282596408,0.12102353951834258,0.09836263774009961,400,12.0,12.0,13.0,12.405 +fresh-en-1,17,18,Q,16,linear_attention,400,0,400,0.11021345909684897,0.338476461940445,0.5653739788569512,0.3405843933156575,400,0.026052736118435862,0.07995821535587311,0.13885354250669482,0.08218945023138076,400,6.162025730218611,8.99159508324555,11.725378448703385,8.919756278077184,400,0.3851266081386632,0.5619746927028468,0.7328361530439615,0.557484767379824,400,0.024304100746185248,0.05196265211061707,0.10643682706639745,0.06018104605127638,400,0.17018972562575307,0.23483492126133199,0.3455615882768044,0.24680718492958292,400,0.46116669656987885,0.5377724947820358,0.6371559538064466,0.543455840799756,400,0.7051735457206023,0.7583936969854026,0.8222305189114208,0.7598993752124531,400,0.059531921278669256,0.08999373682597112,0.11876229663901106,0.08958152678238605,400,11.0,12.0,13.0,12.06 +fresh-en-1,17,18,K,16,linear_attention,400,0,400,0.15282106256345288,0.2809805085416883,0.3931443728040904,0.2805546534269524,400,0.03894457146525383,0.06303644925355911,0.09764739125967026,0.06536894329823553,400,7.3855785843073045,9.157039695880897,11.198219663289356,9.194545587901052,400,0.46159866151920653,0.5723149809925561,0.6998887289555847,0.5746590992438158,400,0.028586844203425207,0.04981955388290905,0.07775894252854917,0.052215458874604034,400,0.18311199374376386,0.23556353868911,0.2917889990107006,0.2369250187215229,400,0.4688927293553796,0.5302337887860165,0.598830161758717,0.5316404632385534,400,0.7062751739942418,0.7549123075430664,0.7949916084244423,0.7523710404851436,400,0.07047217719716797,0.09150435392256773,0.11898176092652953,0.09317170063363989,400,11.0,12.0,13.0,12.2175 +fresh-en-1,17,18,V,16,linear_attention,400,0,400,0.2774945611832664,0.35947241412941366,0.4619440575130284,0.36702827518805864,400,0.08454172611236573,0.11752297729253769,0.152948434650898,0.1176844159886241,400,4.7442536498612,6.612855922287933,9.075520340124022,6.771454527052574,400,0.296515853116325,0.4133034951429958,0.5672200212577514,0.4232159079407859,400,0.05086594555205776,0.09463530374021031,0.1581669652134941,0.09970563673992572,400,0.2417635412718359,0.32240931247017585,0.4163903103217813,0.3248960376505816,400,0.5231054962188049,0.6011278080230325,0.683815776468913,0.603343330494613,400,0.742005256697775,0.7967629643358549,0.8412049897339837,0.794810194985394,400,0.051316181520563414,0.07325260415466169,0.10211204573590976,0.07517182314839194,400,10.0,12.0,13.0,11.505 +fresh-en-1,17,18,QKV,16,linear_attention,400,0,400,0.25511412443593146,0.3471266374690458,0.45071079235058276,0.35192665737500645,400,0.06657342314720154,0.09271714463829994,0.12381824478507043,0.09386549964547157,400,6.329031530953002,8.233270933419504,10.36751110898895,8.237898656319253,400,0.39556447068456263,0.514579433338719,0.6479694443118094,0.5148686660199533,400,0.036218878426732264,0.06288898101822214,0.10186895192560877,0.06701920651092315,400,0.21196174945463794,0.27092794646834495,0.324364166615514,0.2700074613094992,400,0.48644151171128663,0.549135193708908,0.6287772443816939,0.5563556252063723,400,0.7166055126248112,0.7651511031305743,0.8091479921400628,0.7650183684967555,400,0.06448574450155656,0.0875607241866814,0.11268813041886952,0.08823208961729008,400,11.0,12.0,13.0,12.045 +fresh-en-1,18,19,Q,16,full_attention,200,0,200,0.4311848957557231,0.507052164291963,0.6162233195267618,0.5107508661586325,200,0.060883527249097825,0.0817686878144741,0.11165664494037628,0.08477950628846884,200,10.267074556881408,12.053063406863082,13.55564874112228,11.928605507424567,200,0.641692159805088,0.7533164629289426,0.8472280463201425,0.7455378442140355,200,0.012021317325446302,0.021830902157727136,0.03722575109736039,0.02369911731587978,200,0.1305501675613611,0.16279934066108026,0.20521098282814415,0.1658020674863059,200,0.39593018685692877,0.4437525128426434,0.5087399537918276,0.44761150660493654,200,0.6545650693567531,0.6931784178368067,0.7447730326426625,0.6973884541790099,200,0.09329714259774999,0.11905223206513435,0.14710723192008082,0.11925183171336029,200,12.0,13.0,14.0,13.005 +fresh-en-1,18,19,K,16,full_attention,200,0,200,0.22670026507694274,0.30704461922869086,0.4183166562113911,0.3150407258141786,200,0.039362269267439846,0.056449003517627716,0.08684100806713105,0.0604136099293828,200,7.396835134695401,10.406239556138406,12.041620069769985,10.129963886301061,200,0.46230219591846256,0.6503899722586504,0.7526012543606241,0.6331227428938163,200,0.021915179915932743,0.035839364439629914,0.07754615250883176,0.04206697492328054,200,0.14859099883759383,0.18178755308565453,0.2955799235864016,0.1952872559673132,200,0.443392592852635,0.5233448032550203,0.6079653084854294,0.5245941471393899,200,0.7104302998560672,0.7471288199464352,0.7901370207663001,0.7501334067637319,200,0.07855729515968833,0.09740128403048563,0.11067932950297159,0.09652463636806345,200,12.0,12.0,13.0,12.46 +fresh-en-1,18,19,V,16,full_attention,200,0,200,0.19400029163807633,0.26802419405430555,0.7444141902960837,0.34145368648692964,200,0.03089746590703726,0.044242747128009796,0.09369491934776303,0.054221234880387784,200,11.256714352597237,12.289513119609753,13.580736845983463,12.20946932958521,200,0.7035446470373273,0.7680945699756095,0.8487960528739664,0.7630918330990756,200,0.011875960653144007,0.02012824268777999,0.02812072543826031,0.02157557949596511,200,0.13053710679682953,0.15637861453178792,0.1861536813133661,0.16196019077726412,200,0.39005189977281957,0.43376370914656515,0.4878899524965831,0.43821051609594314,200,0.6380715811022685,0.6832696695651446,0.7298022814893581,0.6826428343630974,200,0.105279133664871,0.12693181904971995,0.15821533083162037,0.12834481768224765,200,13.0,13.0,14.0,13.12 +fresh-en-1,18,19,QKV,16,full_attention,200,0,200,0.31665201601572335,0.39338733768090606,0.5794233224354685,0.41706886326428505,200,0.04845853969454765,0.06443605571985245,0.0888820044696331,0.06650541638955473,200,11.007434277440948,12.080045669666244,13.410993294223736,12.089160451072276,200,0.6879646423400593,0.7550028543541403,0.8381870808889835,0.7555725281920173,200,0.01287010285103627,0.02163322757509914,0.030237538393672932,0.02213954175105454,200,0.1312497628212408,0.15828317223401697,0.1958847498207491,0.1603144957098266,200,0.3972756301685534,0.4453810209838822,0.48398942204827167,0.44485647950146784,200,0.6651939623777732,0.6972101794953363,0.7309527729056692,0.6971647468186949,200,0.09977808790465417,0.11870345155728651,0.1370675737754651,0.11832447995976308,200,12.0,13.0,13.0,12.965 +fresh-en-1,18,19,JS,16,full_attention,200,0,200,0.011181799773294188,0.020532311007855242,0.036456327223640984,0.023194204544329865,200,0.004730595322325826,0.011461302638053894,0.025105519965291017,0.014161832590471022,200,1.7412255463230686,2.659619212849171,4.217517466409308,2.8566335462318433,200,0.10882659664519179,0.16622620080307318,0.2635948416505817,0.1785395966394902,200,0.186248787168078,0.3343939097149974,0.5459485804755462,0.3479876147285212,200,0.37021580694925976,0.5612792278356824,0.7436607439631449,0.5598981971998174,200,0.8284835735130546,0.8959170292033419,0.952508250082405,0.8940651980407996,200,0.9457998846249187,0.9721946631458249,0.9865427685131878,0.9696579546258809,200,0.0032790084639732243,0.006570122373840076,0.01298995768820614,0.00743597117095286,200,3.0,5.0,6.0,4.545 +fresh-en-1,18,19,sqrtJS,16,full_attention,200,0,200,0.30193768690805883,0.3825230540242046,0.45287913030479104,0.3826136138779111,200,0.06877774968743325,0.10705734044313431,0.1584443613886833,0.11202324468642473,200,5.280599074492594,7.197478374739278,8.886627395716664,7.092955534662397,200,0.3300374421557871,0.4498423984212049,0.5554142122322915,0.4433097209163998,200,0.05336540868347305,0.0815335800050039,0.13533071359772827,0.09134531188350514,200,0.2066737892973911,0.2802597576393344,0.3744575598529207,0.287352390095275,200,0.5709947479340537,0.636453290661577,0.7013767933232948,0.6394547653851447,200,0.7891912367314837,0.8343033990581601,0.8732837537132458,0.8324792919488169,200,0.045984321058947716,0.05991051438894515,0.07563192302723536,0.06042171254999941,200,10.0,11.0,12.0,10.665 +fresh-en-1,19,20,Q,8,linear_attention,400,0,400,0.1669487931765616,0.3607848957180977,0.6931610012426973,0.410217474353849,400,0.05229693911969662,0.10168685764074326,0.27130539715290075,0.13658672756515444,400,3.8522452858604623,5.371966498725807,6.603127491725998,5.329706811730791,400,0.4815306607325578,0.6714958123407259,0.8253909364657498,0.6662133514663489,400,0.030221015224022017,0.0698875323073061,0.15381598753968753,0.08070423305533554,400,0.2365747761013695,0.3089826393959058,0.43295096518653253,0.3198429092928105,400,0.4089720336405216,0.4922276664695664,0.6179908730739344,0.5058147722473455,400,0.6820258312389749,0.7755111152843578,0.8335079475059852,0.7674403842557654,400,0.04597965340383984,0.08583720637001785,0.1264873408641523,0.08557084511623253,400,5.0,6.0,7.0,6.1625 +fresh-en-1,19,20,K,8,linear_attention,400,0,400,0.24076348687522114,0.37530899699777365,0.5617913661524654,0.39355633352941366,400,0.05804447829723358,0.10762063413858414,0.2099820077419281,0.12182375728152692,400,4.086322977661162,5.506324176655856,6.669362162591686,5.409461825292958,400,0.5107903722076452,0.688290522081982,0.8336702703239608,0.6761827281616197,400,0.028502297388318925,0.06469647797532935,0.13682192031180637,0.07621358134977661,400,0.20792188052625313,0.2839024676300036,0.42101042044324855,0.3033901752094671,400,0.3969658851053383,0.48115138500216087,0.5939299379550633,0.4900535095677916,400,0.6941012592948048,0.7869372845924716,0.8468555717827664,0.7768774297560458,400,0.048694880244755594,0.0803396814938182,0.12319150367942093,0.08427186059809469,400,5.0,6.0,7.0,6.145 +fresh-en-1,19,20,V,8,linear_attention,400,0,400,0.29485651399008933,0.38210237585008144,0.5098383449949324,0.4015845881862333,400,0.07950440570712089,0.10472380369901657,0.14245635718107225,0.10834775455296039,400,4.664195094318613,5.499528094296002,6.575111898001579,5.580150773864391,400,0.5830243867898266,0.6874410117870002,0.8218889872501973,0.6975188467330489,400,0.03095963461312918,0.0649529756526238,0.10217059357767191,0.06581726863174843,400,0.21033675903760674,0.2693376143233327,0.34880853406705975,0.27575926999406536,400,0.3998346753079063,0.48935780717891786,0.5589212354807838,0.484013013274646,400,0.6938323414041523,0.7821831014716134,0.8382690015081081,0.7730889045096074,400,0.046839980923763166,0.07964465367848908,0.12154470969641071,0.08256188418928931,400,6.0,6.0,7.0,6.1725 +fresh-en-1,19,20,QKV,8,linear_attention,400,0,400,0.2895806004293263,0.41380413877777755,0.5908717142418027,0.42657199458801187,400,0.07591332346200944,0.10958828404545784,0.1958449184894562,0.1290421062335372,400,4.4357837395409705,5.53182679381454,6.645874515744391,5.520771682619832,400,0.5544729674426213,0.6914783492268175,0.8307343144680489,0.690096460327479,400,0.029107884071064693,0.06373960453391098,0.11479004728519476,0.07005115593812697,400,0.22216170939544147,0.28768649637532595,0.38790090691655543,0.29597808011186333,400,0.3997630330317304,0.48265860232217883,0.5730654786817836,0.48778681564858106,400,0.6878482048322893,0.779512916738056,0.8374141693449785,0.7709493579211607,400,0.051382973422210085,0.08402486592549027,0.12246324777035782,0.0853742846969992,400,6.0,6.0,7.0,6.2025 +fresh-en-1,20,21,Q,16,linear_attention,400,0,400,0.10556338662281632,0.1702877018833533,0.5006994354538621,0.2359837639743637,400,0.013436282984912395,0.025442440062761307,0.07741060778498651,0.03515632544178516,400,10.39298870288527,12.380351729083086,13.881980716585488,12.252051462896505,400,0.6495617939303294,0.7737719830676929,0.867623794786593,0.7657532164310316,400,0.010171551914208599,0.01949154040685766,0.03596662935133431,0.021689419201811937,400,0.11116455393606761,0.14171180462769878,0.18636323836834381,0.14765098724229075,400,0.3870003806517106,0.4413192794953636,0.5122803529954421,0.44567193247636155,400,0.6548255183310671,0.7014329292618093,0.7596170279911209,0.7065847281339503,400,0.084031583321219,0.11449742305194535,0.1426332172305131,0.11410944509573691,400,12.0,13.0,14.0,12.8325 +fresh-en-1,20,21,K,16,linear_attention,400,0,400,0.1254141354933381,0.31875117239542305,0.4695759161841125,0.31453744126600214,400,0.015699871070683002,0.045685237273573875,0.07883117273449898,0.04893102428875864,400,10.493287817718581,12.323432574516353,13.639358357092167,12.126749688150348,400,0.6558304886074113,0.7702145359072721,0.8524598973182604,0.7579218555093967,400,0.01153838086220905,0.019889338830224618,0.034985729954510957,0.022536353009470687,400,0.11546060425145166,0.14182870106815737,0.1944304074122134,0.14985991516274125,400,0.394215054763074,0.4464655049684253,0.5157728110244876,0.4512684057224494,400,0.6663585935024191,0.7085837816420428,0.7526289974216942,0.7089678009597737,400,0.09081340613406973,0.11619876945044194,0.13910362367582935,0.11538286864432899,400,12.0,13.0,13.0,12.8775 +fresh-en-1,20,21,V,16,linear_attention,400,0,400,0.25217119075823574,0.32624402130022645,0.4832525714300574,0.34900206910388076,400,0.03371674492955208,0.05066070333123207,0.07956316694617271,0.05547551088035107,400,9.845477528903364,11.865905141689801,13.46786708092047,11.765443984716676,400,0.6153423455564603,0.7416190713556126,0.8417416925575294,0.7353402490447922,400,0.012534201979382278,0.02322674343366416,0.04167419692951993,0.025509746670976247,400,0.11845330388847569,0.15247917265409766,0.20862972636857557,0.15833545665699386,400,0.39739705739933784,0.460859824204532,0.5327016368707144,0.46253117447757086,400,0.669278317553092,0.7213211804999263,0.7713056755743141,0.7208012258463148,400,0.0794525392913978,0.10616823729404243,0.1355381088861538,0.10667965292390058,400,12.0,13.0,13.0,12.6275 +fresh-en-1,20,21,QKV,16,linear_attention,400,0,400,0.2158252321649343,0.3058822457678616,0.4426770078018308,0.32152505216770805,400,0.02840803787112236,0.04321477375924587,0.07283187955617906,0.04780787133146078,400,10.695067264134522,12.471487618407613,13.598548378642471,12.253714191361828,400,0.6684417040084076,0.7794679761504758,0.8499092736651545,0.7658571369601143,400,0.011773084417610121,0.018861756473651455,0.03306781498827839,0.02137352325329592,400,0.11477124887498903,0.1399193807328054,0.1897439512600233,0.14763040284353285,400,0.39483599207289943,0.43927910978017354,0.5012091569335229,0.44528549358863173,400,0.6670881935719456,0.7058763958976542,0.7495148225919139,0.706972063091183,400,0.09079675272378648,0.11481036310159395,0.14047209104347333,0.11550730197743743,400,12.0,13.0,13.0,12.88 +fresh-en-1,21,22,Q,16,linear_attention,400,0,400,0.06384462246205659,0.17670703050680459,0.3051427816972137,0.1783992947047227,400,0.008537944965064525,0.019847989082336426,0.04315142333507538,0.02371753485291265,400,11.671821628997062,13.624235699940158,14.597915958091974,13.341410534878078,400,0.7294888518123164,0.8515147312462599,0.9123697473807484,0.8338381584298798,400,0.006403124275131964,0.011625187335285184,0.024721559600750283,0.01419021938387527,400,0.0973642688887752,0.1190732743997841,0.17316350045642953,0.1294566812477966,400,0.34999135350359767,0.38975334941419776,0.45282206139405345,0.3990459855676275,400,0.62320587929238,0.6612313282788462,0.7118820228051657,0.6648889018559109,400,0.10298115201791763,0.1346651935603197,0.16243770176032532,0.13333913666379016,400,13.0,13.0,14.0,13.2625 +fresh-en-1,21,22,K,16,linear_attention,400,0,400,0.07246483792550863,0.22561558743473142,0.35786427110433583,0.22181170293828473,400,0.0084521958604455,0.02611955627799034,0.04820340760052205,0.027845182007877157,400,12.27069476884793,13.88397595736042,14.749479648176408,13.657958988402155,400,0.7669184230529956,0.8677484973350262,0.9218424780110255,0.8536224367751347,400,0.005652270414502424,0.010160509633652256,0.020261310386206936,0.011952356216040237,400,0.09532105401830814,0.11783834080437176,0.16055031323718438,0.12385020293634608,400,0.345382030078494,0.3833532434341703,0.441932155785918,0.38922357247356737,400,0.612796262428149,0.6518430956842234,0.7008728843911339,0.6543664877335331,400,0.11126180936841043,0.14258726219098045,0.166288509203652,0.14055452408609004,400,13.0,13.0,14.0,13.4575 +fresh-en-1,21,22,V,16,linear_attention,400,0,400,0.14645517671015112,0.20409299293532968,0.3581325825769454,0.22789953315776074,400,0.017025454342365267,0.025588324293494225,0.04577934369444848,0.028921679761260748,400,12.08852557688836,13.522871150127056,14.646276087226953,13.412076382157728,400,0.7555328485555225,0.845179446882941,0.9153922554516846,0.838254773884858,400,0.0061618695853761915,0.012212046552993742,0.021571331274238784,0.013407440161870671,400,0.09927100304554601,0.12081479649674734,0.16150543220824096,0.12729382309273476,400,0.34993501766472984,0.39383123961459815,0.44475855450160806,0.3971812251404107,400,0.6220754410744784,0.6634335838380786,0.7090832538792318,0.6650051577631586,400,0.09823440081829309,0.13191439281451312,0.15887990445001454,0.13080731147180194,400,12.0,13.0,14.0,13.2 +fresh-en-1,21,22,QKV,16,linear_attention,400,0,400,0.12880816871766002,0.21573365828953683,0.3277927831746638,0.2218316895258613,400,0.014228214789181948,0.024697483517229557,0.04338475912809373,0.02736534890020266,400,12.39878788299557,13.933333931945104,14.79044721812707,13.729729242531308,400,0.7749242426872232,0.870833370746569,0.9244029511329419,0.8581080776582067,400,0.00545195686218553,0.009888358030941104,0.019363249409110845,0.011459434525819833,400,0.09733048825230976,0.11618109800585436,0.15495674991204322,0.12205374658200668,400,0.3420926328768988,0.38092598778433384,0.4388426519969296,0.3854986705735986,400,0.6127734391661812,0.6500593070293752,0.697628564398646,0.6525462479273656,400,0.10886333608344433,0.14088437661784875,0.1670452230926111,0.13943321976226694,400,13.0,13.0,14.0,13.4325 +fresh-en-1,22,23,Q,16,full_attention,200,0,200,0.3592355742584914,0.4290063641965389,0.5183386398013681,0.4358051378687378,200,0.05295615456998348,0.08061966672539711,0.12658628970384597,0.08682604677975178,200,8.498477804325665,10.389816369128663,12.540282437340775,10.431883194149066,200,0.5311548627703541,0.6493635230705415,0.7837676523337984,0.6519926996343166,200,0.01839256076685949,0.035997972876121255,0.058846210823366635,0.038183476836575574,200,0.13509375279852576,0.19369463025565364,0.263116682752461,0.19706982427250996,200,0.4226903900728656,0.49702955245723746,0.5538745436937531,0.4937323321627609,200,0.6849005184152759,0.7440087642227886,0.7830875008357476,0.7395746941226733,200,0.06609867796105949,0.0894409030977984,0.11773693285425921,0.09111001414597193,200,11.0,12.0,13.0,12.155 +fresh-en-1,22,23,K,16,full_attention,200,0,200,0.32855844057630745,0.405355604365468,0.49803007435984903,0.4135330396937206,200,0.07885837256908418,0.10642682388424873,0.17981866300106045,0.11188872553408145,200,5.406684024131497,7.824861847479696,9.994388355191575,7.962303150587487,200,0.33791775150821857,0.489053865467481,0.6246492721994734,0.4976439469117179,200,0.04006673833221058,0.06967575043832355,0.13066206326311705,0.07497381053769142,200,0.19550840055564703,0.2703558381079888,0.37832679497243243,0.2741570887804091,200,0.5171382798255001,0.5720287824146004,0.6373187901905935,0.569206297494602,200,0.7475374164771328,0.7864549121757407,0.8264046719552145,0.7851415259666082,200,0.05528499658551932,0.06977050443179608,0.0807711393336097,0.07044546433674666,200,11.0,11.0,12.0,11.48 +fresh-en-1,22,23,V,16,full_attention,200,0,200,0.18909952223766596,0.25209691270720214,0.3402364259818569,0.2632962192245759,200,0.03445771336555481,0.06500118225812912,0.14765326529741285,0.08498518273234368,200,3.7907766066634667,7.604634218782142,11.037576690480968,7.432812717686302,200,0.23692353791646667,0.47528963867388385,0.6898485431550605,0.46455079485539386,200,0.029973636053918434,0.07359972122606995,0.21477279383423586,0.1024102973704517,200,0.16090193943222877,0.28487560712478993,0.49306594634220835,0.30398565210836764,200,0.49545155390891044,0.5872090868164739,0.6985055282853629,0.5938313081564036,200,0.7466452297397497,0.8175944880420538,0.8688542602800692,0.8086692862370422,200,0.0410416809296553,0.05915167581360657,0.088387367754954,0.061955791457909866,200,9.900000000000002,11.0,12.0,10.92 +fresh-en-1,22,23,QKV,16,full_attention,200,0,200,0.3165625500259921,0.3903192594880238,0.4747440696693957,0.39382400568167214,200,0.05849056728184223,0.08628058061003685,0.1325710192322731,0.0935366622172296,200,6.609176746232382,9.043481104332157,11.62641145284913,9.02073667887968,200,0.4130735466395239,0.5652175690207598,0.7266507158030706,0.56379604242998,200,0.025078482539711726,0.05128207313746439,0.09472850067847781,0.05787633461985751,200,0.1533327697107659,0.22689142102930943,0.33277540974781106,0.2369125603220013,200,0.46867103693328954,0.538736420168326,0.605079627042817,0.5383087529349373,200,0.7142758519297637,0.7724137837943109,0.8124701520823968,0.7661928601375356,200,0.05926498734040882,0.0789352885292616,0.09938865574208369,0.08012805172165825,200,11.0,12.0,12.099999999999994,11.81 +fresh-en-1,22,23,JS,16,full_attention,200,0,200,0.0006548373678775478,0.005710974774956412,0.010046254622852756,0.005727734705190684,200,0.0003446378308581188,0.002191954874433577,0.005025189882144332,0.0028395897474547383,200,2.0805880696920287,3.456884827849072,4.795167555360664,3.4264856378474575,200,0.1300367543557518,0.216055301740567,0.2996979722100415,0.2141553523654661,200,0.15577992324836368,0.24189662990558702,0.44600929059296407,0.2787465944918049,200,0.31801049757627503,0.4574181097015638,0.6713400086388607,0.4804506375313599,200,0.7890561926977303,0.8653031491092444,0.93487827550346,0.8635734485331668,200,0.9305897464303312,0.9601534309674675,0.9821965403596922,0.9579355198906288,200,0.003565159552665757,0.008419251055247715,0.01576835887178605,0.009210318846961947,200,4.0,5.0,7.0,5.425 +fresh-en-1,22,23,sqrtJS,16,full_attention,200,0,200,0.06319582018768415,0.21820172836305574,0.2689065179089084,0.19664797570407244,200,0.018564413674175738,0.046818315982818604,0.07088856622576714,0.048208457650616765,200,6.231787440009218,8.087427799617846,9.606946661964134,7.942121688052594,200,0.38948671500057613,0.5054642374761154,0.6004341663727584,0.4963826055032871,200,0.04436429250269472,0.06522627446553322,0.10450701511879815,0.07312091215983446,200,0.18335785834009402,0.23800731424651156,0.32609806366392924,0.24993717399838947,200,0.5431624918329219,0.6045795630472731,0.6805249844030334,0.6099194619996151,200,0.773016331414066,0.8156516430145131,0.8662197142008561,0.8177819383249778,200,0.042215155369451175,0.06066362336116296,0.08112226200805865,0.06092651660864405,200,10.0,11.0,12.0,10.86 +fresh-en-2,0,1,Q,16,linear_attention,400,0,400,2.4726752978749573,3.7375750923529267,4.850858685933053,3.6861189761641437,400,0.46132279336452486,0.8543250560760498,1.1934852480888367,0.8383005390316248,400,6.370148327369052,8.569242898757494,10.582774403795472,8.509324284369315,400,0.39813427046056576,0.5355776811723434,0.661423400237217,0.5318327677730822,400,0.034126061746137866,0.05780951991934226,0.1007820661802688,0.06362601097720445,400,0.1641044477471998,0.22125739735807703,0.30661290944083297,0.2265869632546195,400,0.5068489897298353,0.5937344584591551,0.6911788683528843,0.5988805846990324,400,0.7512344768410083,0.8079303279028176,0.8646909538119537,0.8071989728185577,400,0.04174391912420662,0.06480786376597439,0.09439553022675148,0.06659055561126649,400,10.0,11.0,12.0,11.115 +fresh-en-2,0,1,K,16,linear_attention,400,0,400,2.9589993285480887,3.6147952508181334,4.718218704406172,3.7445520730968567,400,0.4922227650880814,0.7032828629016876,0.9911369681358339,0.7268326603621245,400,7.514437478258538,9.898047495738098,11.573468821562177,9.685346151940486,400,0.4696523423911586,0.6186279684836311,0.723341801347636,0.6053341344962804,400,0.025498166889913056,0.041098731578761385,0.07528234231793031,0.0467694330856743,400,0.14839073747428455,0.18401677417925733,0.2528621414643858,0.19402203840318355,400,0.4637833956027394,0.5404437625709599,0.6572332192339179,0.5527928895103493,400,0.725146747672929,0.7744256398733314,0.8421464561317874,0.7802984129646038,400,0.04984795481484179,0.07412155184761895,0.09673434123797693,0.07391592663543128,400,10.900000000000006,12.0,12.0,11.5675 +fresh-en-2,0,1,V,16,linear_attention,400,0,400,1.601429042359814,2.718538397923112,4.425311133172363,2.9363659172540064,400,0.3051665127277374,0.5337409377098083,1.0304036021232605,0.6539870547875762,400,6.511655664285719,8.95473583228312,11.4190294810497,8.917105365731068,400,0.40697847901785744,0.559670989517695,0.7136893425656062,0.5573190853581917,400,0.026744660791879857,0.05245099216047753,0.09714217914207937,0.05869019282004693,400,0.1530797588129219,0.20582640260625115,0.29660369427235606,0.2165936337414439,400,0.47062371731904123,0.5715773292677548,0.6824480474656865,0.5758182051673303,400,0.7402655687431883,0.8064928996490706,0.8758530033052949,0.8059241440180978,400,0.03513627097313222,0.05950396458654637,0.08867234972764908,0.060757833076099776,400,9.0,11.0,12.0,10.9 +fresh-en-2,0,1,QKV,16,linear_attention,400,0,400,2.877012618817389,3.5474699581973255,4.493177025020123,3.66330001636059,400,0.4910459160804749,0.7071874737739563,0.9930141746997834,0.7431290470808745,400,7.605499827453808,9.428086651037585,11.062832651678335,9.378469791354695,400,0.475343739215863,0.5892554156898491,0.6914270407298959,0.5861543619596684,400,0.02975228032444027,0.04647046364289961,0.07358272721981475,0.04946543302789688,400,0.15542508394133997,0.19372789644610697,0.2510150381644268,0.20077961029927172,400,0.4938348442305548,0.5556642334671317,0.6580205748407775,0.5663213427535048,400,0.7366804415922553,0.7830993735245134,0.8439915024416458,0.7875184033498662,400,0.04918868848191448,0.07251381658806932,0.09602754578482219,0.0732631741612165,400,10.0,12.0,12.0,11.49 +fresh-en-2,1,2,Q,16,linear_attention,400,0,400,0.6467387733049691,1.2311389315873384,1.8427374650724233,1.238258186922467,400,0.11037619113922119,0.22918583452701569,0.37181766927242277,0.23878039263188838,400,7.666329272037907,9.810243845300857,11.781536222731383,9.707409332745465,400,0.4791455795023692,0.6131402403313035,0.7363460139207114,0.6067130832965916,400,0.02387048511732007,0.04206322101995855,0.07247004450019576,0.04660961999145874,400,0.13632660638494107,0.18128874868907635,0.25978036721515263,0.19189095232877765,400,0.45573278083647095,0.5323556112577286,0.6308779086002292,0.5381222745281744,400,0.7333901962123509,0.7991498825958216,0.8738924153588391,0.8032625103647323,400,0.03089682737632073,0.06159943465153084,0.09036398803526896,0.06147530041301772,400,9.0,11.0,12.0,10.855 +fresh-en-2,1,2,K,16,linear_attention,400,0,400,0.8913048025220632,1.256248516496271,1.8419503295794135,1.317925817295909,400,0.13660329878330232,0.21154297143220901,0.3379586696624756,0.22862691062502563,400,8.447188401575257,10.739624947687648,12.042823060074989,10.468261262682963,400,0.5279492750984536,0.671226559230478,0.7526764412546868,0.6542663289176852,400,0.021906145052908697,0.03265399671790962,0.05960815708481322,0.03746161168453016,400,0.13342658054123857,0.16530705564374365,0.22179004827313162,0.1722901887717966,400,0.4469838121164075,0.4976041223841733,0.5964288855435161,0.5108754707313077,400,0.7228842970993927,0.7712707521745725,0.8696657159262754,0.7834203764667115,400,0.03288613354617825,0.07264717896387868,0.09801402770291148,0.07015424052433737,400,9.0,12.0,12.0,11.2625 +fresh-en-2,1,2,V,16,linear_attention,400,0,400,0.5403325803228655,1.1034665680490434,2.0374572062864904,1.2351520844992774,400,0.09745831266045571,0.20665869861841202,0.41894162297248844,0.2381142215244472,400,7.318665325830582,9.895466420169896,11.806494366824161,9.702305093905842,400,0.4574165828644114,0.6184666512606185,0.7379058979265101,0.6063940683691151,400,0.023679125423100085,0.04112691671292894,0.07907946082274686,0.04763484476235912,400,0.13688282687279663,0.182339471738797,0.2641602463956752,0.19311368117776517,400,0.4534282422636782,0.5283937569638961,0.6366183464678434,0.5384767898898049,400,0.7327250310213732,0.7925347524700868,0.8910489723762006,0.8041736424847039,400,0.024614203013904705,0.0644507245582922,0.09088542137492452,0.06024673015478619,400,9.0,11.0,12.0,10.795 +fresh-en-2,1,2,QKV,16,linear_attention,400,0,400,0.878587561333552,1.2940925732254982,1.877209843322635,1.346116238751565,400,0.1360565632581711,0.21797826886177063,0.34693442583084116,0.22969867292791604,400,8.621025371466557,10.694252395723801,12.212494194587105,10.510285401402925,400,0.5388140857166598,0.6683907747327376,0.7632808871616941,0.6568928375876828,400,0.020675578964494037,0.033075438937304795,0.057062049988076555,0.036740563565140695,400,0.12957713084095532,0.1647783320134272,0.2172512957660002,0.1702304113937264,400,0.43987812749485644,0.49825836045590266,0.5864941104902756,0.5071068021861384,400,0.7262666567270635,0.776021957305673,0.8709535333453802,0.7840463441972022,400,0.03214527515774681,0.07192751393777477,0.09553289445336813,0.06913423036648955,400,9.0,11.0,12.0,11.2275 +fresh-en-2,2,3,Q,16,full_attention,200,0,200,1.0245461883954705,1.468230809085071,1.870929591357708,1.4721055381374026,200,0.27064292430877684,0.3516651839017868,0.4466122210025787,0.35912626929581165,200,6.220617909210992,8.74608467859909,11.104585684205976,8.723040167139187,200,0.388788619325687,0.5466302924124431,0.6940366052628735,0.5451900104461992,200,0.029389973000521926,0.055292712329868564,0.10480621041744652,0.06194972526936301,200,0.18119372125826563,0.2407592321613739,0.340973026424329,0.2529695794603423,200,0.47287612193373557,0.5388058863535494,0.6219638667939771,0.5442941717340591,200,0.7370295036279807,0.7769191257507782,0.8712316291882225,0.7921125460713233,200,0.02097259430022871,0.06569261947406238,0.09258158458521229,0.059666472932346594,200,9.0,11.0,12.0,11.055 +fresh-en-2,2,3,K,16,full_attention,200,0,200,1.047112238497357,1.3830288252793252,1.6823188211303202,1.3870920732570813,200,0.2405281513929367,0.30231399834156036,0.3850120991468429,0.3106417492032051,200,7.2917843396224,9.400851595908183,10.8279290061043,9.210093915964974,200,0.4557365212264,0.5875532247442614,0.6767455628815188,0.5756308697478109,200,0.031845273768464966,0.0467995407937716,0.0796240975505653,0.05189737358849848,200,0.18319685191509907,0.22729200392876242,0.2810025984757502,0.22800687197114072,200,0.47485990545348766,0.5331804342386007,0.615801808437866,0.5416787462808252,200,0.7361404674228293,0.7790113353487463,0.8596767360412226,0.7921691180448177,200,0.023651014209028386,0.06261707083707935,0.08731257263524209,0.05803785389371713,200,9.900000000000002,11.0,12.0,10.94 +fresh-en-2,2,3,V,16,full_attention,200,0,200,1.319590831536334,1.6047782362438738,2.436248807935043,1.7316110165239662,200,0.28553804755210876,0.34063076972961426,0.4763486683368682,0.3643722167611122,200,7.714245961968656,10.004611715539575,11.033487342111782,9.607310112547317,200,0.482140372623041,0.6252882322212234,0.6895929588819864,0.6004568820342073,200,0.030009263889161565,0.03995084165326538,0.07162707373498824,0.04689369341071964,200,0.17848458070015924,0.20853301539657854,0.27018445710617073,0.21590573780832556,200,0.4737672318727135,0.5158876035964943,0.5829604398099849,0.5267435143700123,200,0.7354610994158117,0.7717946942656582,0.848727381471732,0.7834530778144226,200,0.02437345767900418,0.0740901848620152,0.09869208400377266,0.06531558853789118,200,10.0,12.0,12.0,11.26 +fresh-en-2,2,3,QKV,16,full_attention,200,0,200,1.1746613873168825,1.510028968565166,2.0858569553587585,1.555826693035633,200,0.28438410758972166,0.3382027745246887,0.4241101145744324,0.3472541347891092,200,7.2575944999530355,9.508067081538398,11.0402235143104,9.32555196008267,200,0.4535996562470647,0.5942541925961499,0.6900139696444,0.5828469975051669,200,0.02994973474423352,0.04551877659702507,0.08030585122282001,0.0507103968599783,200,0.18697113209949143,0.223359743917515,0.2870124131716494,0.22866373544002264,200,0.46995368469248416,0.5211731689492343,0.5982711448874869,0.5313307769316011,200,0.7317361846740532,0.7699960939205248,0.8574193990555813,0.7850918204550286,200,0.023050843473959227,0.07008059645349982,0.09458089018701459,0.0627653209542192,200,10.0,12.0,12.0,11.15 +fresh-en-2,2,3,JS,16,full_attention,200,0,200,0.04602197141357465,0.07628454504492765,0.1473817919920748,0.08694661521343278,200,0.018520138412714004,0.037362802773714066,0.10735419690608977,0.05066753458697349,200,1.4161641426749698,3.258386667467023,5.506307537049726,3.432265370737158,200,0.08851025891718561,0.20364916671668895,0.3441442210656079,0.21451658567107237,200,0.12705090325975432,0.2606942211190351,0.6865422169867628,0.32840330493091907,200,0.32065657972711803,0.5216855022234688,0.8383881975030106,0.5430453325998795,200,0.7088595794927229,0.8122219251794255,0.9368725031561941,0.8176237296849093,200,0.8909261078428391,0.9403066200847392,0.97681748180342,0.9366664684168806,200,0.0049321402912174805,0.013636212932307625,0.02948606380393602,0.01560636028815923,200,3.0,7.0,9.0,6.28 +fresh-en-2,2,3,sqrtJS,16,full_attention,200,0,200,0.619015127280727,0.8163227243348956,1.0459965398535132,0.8165744480350986,200,0.13608818650245666,0.1932932659983635,0.32764911055564877,0.21182226568460463,200,5.032549758296777,9.057259810463414,10.919899757335635,8.51093830816483,200,0.31453435989354855,0.5660787381539634,0.6824937348334772,0.5319336442603019,200,0.03101657788279797,0.05110258394874739,0.1452888043355792,0.06963123683933645,200,0.1743225468570771,0.2408677450329705,0.40932954352313083,0.2626818735898847,200,0.47965442934278435,0.5493611268914893,0.6477410004917799,0.5567935643343007,200,0.7380118569484899,0.7810466112897226,0.8285687368273159,0.7808712610072976,200,0.05596665307328854,0.07573437118325474,0.09884046993322032,0.07700475198656906,200,11.0,12.0,12.0,11.61 +fresh-en-2,3,4,Q,8,linear_attention,400,0,400,0.6649561457335949,1.219953540712595,2.6155741229653366,1.4194106929237023,400,0.12555328309535982,0.2721608132123947,0.9420243918895721,0.3764429011940956,400,4.342626532963589,6.78315662109759,7.37305452280333,6.434599875350081,400,0.5428283166204486,0.8478945776371988,0.9216318153504163,0.8043249844187601,400,0.012147427648664935,0.02562745062251169,0.1203167404662752,0.0421885130965534,400,0.18393789221082077,0.22123033414880283,0.34739905509138547,0.2430227498164566,400,0.34435490376934663,0.39991197067760337,0.6099910540013264,0.4318689205754995,400,0.6180068245846253,0.6682703104159602,0.8193543337700204,0.6833205233169053,400,0.07401470150915074,0.13653564529018658,0.1705915605731225,0.13170581241029286,400,6.0,7.0,7.0,6.83 +fresh-en-2,3,4,K,8,linear_attention,400,0,400,0.7907507222145796,1.0601140707731247,1.778627359867096,1.179323242297396,400,0.14650939404964447,0.22052498906850815,0.4083239674568177,0.2603167027235031,400,5.886050009413602,7.128247930683139,7.540898162097042,6.914662869931459,400,0.7357562511767003,0.8910309913353924,0.9426122702621302,0.8643328587414324,400,0.008697375319168928,0.017470938767710283,0.05130655890112122,0.024679419176510172,400,0.16954972910896252,0.2015994590747333,0.2871624720706391,0.21593759971974952,400,0.32622308055882765,0.36873658603901566,0.4859077896777362,0.386379829688384,400,0.5964214954438735,0.638564818487333,0.7078618772415729,0.6473389019138014,400,0.11623127697520029,0.14770897092565854,0.17734121500350544,0.14731616216919974,400,7.0,7.0,7.0,6.98 +fresh-en-2,3,4,V,8,linear_attention,400,0,400,0.6409734182059765,1.1392776817083359,2.0076001793146134,1.262079943222925,400,0.15022560954093933,0.28189654648303986,0.511130201816559,0.3171920324675739,400,5.362236076145741,6.40234703545636,7.178733999284263,6.32240716531783,400,0.6702795095182176,0.800293379432045,0.8973417499105328,0.7903008956647287,400,0.016343231888317926,0.035648936230736325,0.07027356911442384,0.0404301325382454,400,0.1885725024063895,0.24561921844135487,0.32716741924271,0.2513250867221497,400,0.35794038360670954,0.42953137045847345,0.5204793817786948,0.43472537312952186,400,0.6342681062884282,0.6932246356319096,0.7546039497955589,0.6929044262707964,400,0.09024454040332917,0.12160453726992959,0.1563652022050162,0.12285383392297845,400,6.0,7.0,7.0,6.7975 +fresh-en-2,3,4,QKV,8,linear_attention,400,0,400,0.8190693102777005,1.2374368868768215,2.0703954823315143,1.3594063983578235,400,0.16075235605239868,0.2730579078197479,0.6301734447479249,0.3255896874144673,400,4.959206473016802,6.922076403074476,7.408050546007524,6.630065698013279,400,0.6199008091271002,0.8652595503843095,0.9260063182509405,0.8287582122516599,400,0.011415176960488212,0.022246092054211717,0.08759449033298618,0.03370551922904639,400,0.17829053065297845,0.21636819156727227,0.31219373655668065,0.2306513543269615,400,0.33929205434070236,0.3913489735529303,0.5654442385400866,0.41544940221099624,400,0.610030256648355,0.6560877196120638,0.7783386092981099,0.6686280734096068,400,0.09340406401752387,0.14042774014893095,0.17038689285478967,0.1382610003952424,400,6.0,7.0,7.0,6.88 +fresh-en-2,4,5,Q,16,linear_attention,400,0,400,0.725871973251924,1.0927817510673776,1.5961486818268895,1.1253464498047834,400,0.10975784584879877,0.18688800185918808,0.2976639986038208,0.1973886680416763,400,9.144606080305957,11.040169195289979,12.496462929297083,10.866760702057006,400,0.5715378800191223,0.6900105747056237,0.7810289330810677,0.6791725438785629,400,0.018690875850746854,0.029950210817585296,0.04997799682419388,0.03326568349847979,400,0.13244567607840554,0.16490335915965082,0.2269102612737338,0.17379559340795736,400,0.42736637948507855,0.48774343052685587,0.549301814980305,0.490055095945324,400,0.7045730417174718,0.7512496564513784,0.7940561148582901,0.7505738880845146,400,0.05455163907838704,0.07978707782077019,0.10669837238701026,0.08015655235566392,400,11.0,12.0,13.0,11.885 +fresh-en-2,4,5,K,16,linear_attention,400,0,400,0.7027743351995014,1.0270605832338333,1.4451884102076293,1.0504304503949242,400,0.0982924208045006,0.158344566822052,0.25590928792953493,0.1688550974894315,400,10.099886239607773,11.599622033851986,12.989180781691285,11.5346830629833,400,0.6312428899754858,0.7249763771157491,0.8118237988557053,0.7209176914364562,400,0.015452961870328416,0.02529037138910336,0.038945084216842604,0.026745010809668623,400,0.12199724220370352,0.15624526801278849,0.2039877630209418,0.15954533835265047,400,0.41387715423113053,0.4634863119309324,0.5152386558045284,0.46594189616232984,400,0.6849287187239039,0.729512620392049,0.7682730044586318,0.7295995038739145,400,0.06233885228403613,0.09175859392497465,0.11921372220079468,0.09076558242948948,400,11.0,12.0,13.0,12.19 +fresh-en-2,4,5,V,16,linear_attention,400,0,400,0.3912558341631666,0.7567285923287272,1.4041822212748232,0.8495371565574896,400,0.0663964070379734,0.12796133011579514,0.2679863154888154,0.1549627908412367,400,8.374620571223424,10.606467197484676,12.448486754287075,10.463980054535737,400,0.523413785701464,0.6629041998427923,0.7780304221429422,0.6539987534084836,400,0.019019787686590774,0.033900906573831394,0.06070305565304318,0.03813133572886742,400,0.12892018755390666,0.1717914621526032,0.24626076836233604,0.18104937137524169,400,0.4349675972836492,0.5063901590932871,0.5820374696287731,0.5067521537595934,400,0.7032583166341785,0.7617751636612572,0.8195526071417653,0.7621059172178071,400,0.04370974010358383,0.07521794210632365,0.10608964861710969,0.07601459659773907,400,11.0,12.0,13.0,11.6975 +fresh-en-2,4,5,QKV,16,linear_attention,400,0,400,0.7460156418848783,1.031875939690508,1.4543714209692553,1.0767503266842686,400,0.10474185049533845,0.1573282852768898,0.24366632252931594,0.16944135958328843,400,10.207025642508965,11.713868773294205,12.859551962901216,11.613506462098563,400,0.6379391026568103,0.7321167983308878,0.803721997681326,0.7258441538811602,400,0.01628077823305949,0.024393484965163602,0.037836518075872974,0.026007124251734917,400,0.12201895770099955,0.15177858648333475,0.19615813002495547,0.1557016226925138,400,0.41793552184923105,0.4626178792637087,0.5167179543600329,0.46530940687917977,400,0.688528433931907,0.7289389029787537,0.7678191916111183,0.7286986550684604,400,0.06668354919908978,0.09001803136812575,0.11782002194271543,0.0915935307849647,400,12.0,12.0,13.0,12.255 +fresh-en-2,5,6,Q,16,linear_attention,400,0,400,0.851574399881065,1.2328358069062233,1.5754466176033022,1.2165507366415114,400,0.11595259755849838,0.17790406197309494,0.2672141402959824,0.18582824332639575,400,10.144826996279729,12.008939442383436,13.365911937171992,11.824821217034492,400,0.6340516872674831,0.7505587151489648,0.8353694960732495,0.7390513260646557,400,0.013138349574289217,0.022156076450375393,0.03847724505680397,0.024922356338413044,400,0.11762685243736877,0.1454348504320823,0.19613596622563284,0.1532684082947824,400,0.40208604723231467,0.4542885664670485,0.5241036642734235,0.4605019504457323,400,0.6766171783356322,0.7193171977320212,0.7668489052364801,0.7212409207161873,400,0.07826354912288486,0.10406458114392117,0.13074453552784537,0.10453888194747897,400,12.0,13.0,13.0,12.59 +fresh-en-2,5,6,K,16,linear_attention,400,0,400,0.9205419819802045,1.1929860822856426,1.6117078525945545,1.232341660541133,400,0.11639174073934555,0.1654318869113922,0.24083938598632812,0.1722104798257351,400,11.2425955977831,12.841281073341943,14.014298838035334,12.693759180085504,400,0.7026622248614437,0.8025800670838714,0.8758936773772084,0.793359948755344,400,0.009446074381172054,0.016398801405100253,0.028210600112351745,0.018032661692684072,400,0.10973146554507106,0.13357254108664002,0.17607224719933948,0.13933373904712135,400,0.380094123444762,0.4251938429473503,0.4847483025007229,0.4290067602332053,400,0.6481372226254637,0.6922385834963585,0.735282831410127,0.6913632434448346,400,0.09058379240300399,0.12305143931827142,0.1516150179540596,0.12143871867012197,400,12.0,13.0,14.0,13.005 +fresh-en-2,5,6,V,16,linear_attention,400,0,400,0.5877458011265845,1.0415548998862505,1.6692363888956607,1.0993071420257912,400,0.08654429689049721,0.16079697757959366,0.28185429871082307,0.1784923622198403,400,9.666959968871874,11.661491825981747,13.450360318454951,11.570768604118362,400,0.6041849980544921,0.7288432391238592,0.8406475199034344,0.7231730377573976,400,0.012637319389519576,0.0248027543790892,0.04367492999764239,0.027166561649826143,400,0.11794271700146419,0.15820390294402253,0.2139079334873493,0.163147184023073,400,0.3962766279875749,0.46450052856377977,0.5360973813516796,0.4648366761414968,400,0.6663316704905683,0.7227860066702889,0.7806851775714538,0.7229276692082195,400,0.06730682886119302,0.10217063375077115,0.1356192559712031,0.10113584053353626,400,11.0,13.0,13.0,12.47 +fresh-en-2,5,6,QKV,16,linear_attention,400,0,400,0.9811203271150589,1.241085309535265,1.5243493417277933,1.2490848219452892,400,0.1201214000582695,0.1651734933257103,0.22961021214723587,0.17032992394641042,400,11.593473803256467,12.887066835429433,13.868540450188148,12.765544117641005,400,0.7245921127035292,0.8054416772143396,0.8667837781367592,0.7978465073525628,400,0.010246019390033518,0.01610368278507807,0.02533929999020915,0.017375070757173147,400,0.11095594144439093,0.1312068290102344,0.16660866388975967,0.13606221543516797,400,0.3856590041449349,0.4242404428273526,0.47629094480940665,0.42769903019994243,400,0.651606565151038,0.6906464399673401,0.7311633193461577,0.691344675342286,400,0.09264199161897599,0.12293169084725383,0.146209308056351,0.12200012320293563,400,12.0,13.0,14.0,13.035 +fresh-en-2,6,7,Q,16,full_attention,200,0,200,0.7084877190412954,0.891278566326946,1.190908943209797,0.9346836907370016,200,0.14625512659549714,0.1908891573548317,0.23112499713897705,0.19082701101899147,200,7.430946544754058,9.826049778536241,11.886749461918031,9.778457250033208,200,0.46443415904712865,0.6141281111585151,0.7429218413698769,0.6111535781270755,200,0.02306910792070793,0.04188844713478518,0.07687796072289552,0.0459701504999428,200,0.14504629658108995,0.20955932631660323,0.29963729855234367,0.2124838574456277,200,0.45324123542547307,0.5194880072347068,0.5878879896926795,0.5194799418647656,200,0.7280539693123357,0.7787105701394436,0.8184943264888118,0.7755183294087458,200,0.046265510909362914,0.06914927135246665,0.09332641915006186,0.06955375247738518,200,11.0,11.0,12.0,11.465 +fresh-en-2,6,7,K,16,full_attention,200,0,200,0.7711761617567391,0.9527064354624599,1.1831206265371292,0.9592799371527508,200,0.13922533094882966,0.17159172892570496,0.21165581047534943,0.17055463761091233,200,8.95639531920853,10.630897386457768,12.00468645647468,10.573144460786565,200,0.5597747074505331,0.6644310866536105,0.7502929035296675,0.6608215287991603,200,0.022187558960486683,0.033675476373778906,0.05243177549838167,0.03548082048283872,200,0.13912682933089573,0.16810065736711596,0.2383755651750977,0.1818558068230242,200,0.4479396457250513,0.4999939116488935,0.5482019217638111,0.499304907322295,200,0.7240805969317379,0.7683510928462524,0.8017344645478868,0.7637475068882661,200,0.050507847914963024,0.07618105889748847,0.09610291650551317,0.07379344110875108,200,11.0,12.0,12.0,11.64 +fresh-en-2,6,7,V,16,full_attention,200,0,200,0.965393504453823,1.2215298768132925,1.4691833076998586,1.2404682144988328,200,0.16483497470617295,0.21228660643100739,0.25303093791007997,0.21185518980026244,200,8.799200835959866,10.940147770778243,12.439363305464369,10.840030299488305,200,0.5499500522474916,0.6837592356736402,0.777460206591523,0.677501893718019,200,0.019082667602235537,0.03083357565031819,0.054588137943556374,0.03318183728543704,200,0.13517085108942417,0.17056743183901052,0.2416841350071954,0.17545959436426078,200,0.4330909404135759,0.49026814352583253,0.5534017136700364,0.49009559399154495,200,0.7151451145593026,0.7577580225714826,0.7910534176273512,0.7547349806654077,200,0.05646891683339313,0.0797621571999549,0.10362217799586106,0.07947750627937097,200,11.0,12.0,13.0,11.9 +fresh-en-2,6,7,QKV,16,full_attention,200,0,200,0.8439604148734361,1.0644298213301226,1.261562438122928,1.0658534347102977,200,0.1528593808412552,0.19050481170415878,0.22889656126499175,0.18935274064540863,200,8.836788695124728,10.754828477231866,12.225580782317998,10.629566412326012,200,0.5522992934452955,0.6721767798269916,0.7640987988948749,0.6643479007703758,200,0.02058208541327704,0.03251359570599585,0.05404294824057522,0.035148194787918224,200,0.13284157694416462,0.1788917007838831,0.24630757684507618,0.18245421864570616,200,0.44224690712953507,0.4957507159009583,0.5472988236519651,0.49561188525174443,200,0.7187324301286275,0.7608959973387576,0.7927744143058353,0.7594865973331963,200,0.05245626007919535,0.07794464082226368,0.09899585052478131,0.07606100718288497,200,11.0,12.0,12.0,11.755 +fresh-en-2,6,7,JS,16,full_attention,200,0,200,0.02392248858304811,0.04078545750178364,0.07289505623994046,0.04433911242503797,200,0.006813149619847536,0.012385479174554348,0.033921096846461295,0.0166567829658743,200,2.2212617656104263,5.472744621966769,7.795142491361117,5.2085862491113835,200,0.13882886035065164,0.3420465388729231,0.48719640571006984,0.32553664056946147,200,0.07017116877961757,0.12823873508735031,0.41354198376766177,0.18576687870653763,200,0.21498545207837663,0.3169572328970117,0.6440030615827387,0.36982071258550403,200,0.6315613984189503,0.7645023129881782,0.9087314605696031,0.7661572128480736,200,0.8797246356687405,0.9362302054984842,0.9748901003878205,0.9285459982454779,200,0.005088591712897109,0.013080652225355354,0.029537841067918073,0.016327182162954226,200,4.0,7.0,9.0,6.975 +fresh-en-2,6,7,sqrtJS,16,full_attention,200,0,200,0.4461918576504104,0.6388791813515127,0.8278976137051358,0.6368944962022942,200,0.08254119008779526,0.11129002645611763,0.18417678624391556,0.12279642166569829,200,6.93346311189749,10.118440127963417,11.935910340242762,9.699909515080263,200,0.43334144449359313,0.6324025079977136,0.7459943962651726,0.6062443446925164,200,0.022699839718099902,0.03875150949571206,0.08717669813312172,0.049379957173508314,200,0.133885100412968,0.17813627303394444,0.3036569467751457,0.19936827184516573,200,0.45232076974069135,0.5352846335625829,0.639442797795787,0.5414969846474517,200,0.7297303629015461,0.7925243460125426,0.8421283591605325,0.7870725151767829,200,0.04920873150533187,0.06861594240979486,0.09464068989405404,0.07117687340126778,200,10.0,11.0,12.0,11.355 +fresh-en-2,7,8,Q,8,linear_attention,400,0,400,0.9475403716787696,1.5199985392391682,2.848604653775692,1.7078595669660717,400,0.21024447530508042,0.33023060858249664,0.6954267799854279,0.3863687488436699,400,5.973736047039369,6.550707073679574,7.141183501729546,6.552726626284552,400,0.7467170058799211,0.8188383842099467,0.8926479377161932,0.819090828285569,400,0.01718036458227772,0.03160603047934095,0.04845670865572198,0.03245687391515133,400,0.18596667118917806,0.22121750133592655,0.2773823498721183,0.22682021241034392,400,0.35483474844208424,0.4061881828475223,0.45742931466400816,0.40664474310705456,400,0.6399590823657548,0.6916321935628398,0.743822165988582,0.69155443727545,400,0.08928886839364199,0.11413019781625386,0.14220257878763606,0.11534609467094653,400,6.0,7.0,7.0,6.76 +fresh-en-2,7,8,K,8,linear_attention,400,0,400,0.27919708881527183,1.2571341320872307,2.025203455239535,1.2936672358692158,400,0.05680582076311113,0.26616765558719635,0.4585825264453888,0.2758914006408304,400,6.0886081299349115,6.763737825909624,7.281722084110023,6.719104450220252,400,0.7610760162418639,0.845467228238703,0.9102152605137529,0.8398880562775315,400,0.01409160583417988,0.026111136705904134,0.044847044451208774,0.02809281506659452,400,0.17946903723998078,0.209847332683745,0.25088347108160275,0.21325637751392498,400,0.34328151625327247,0.3883191772211009,0.44634769264310353,0.39326432352075824,400,0.627628318180166,0.6808509069008989,0.7419748195123717,0.6833779277095695,400,0.09511912369629705,0.1189512207006724,0.15235469143409813,0.12147274598106943,400,6.0,7.0,7.0,6.815 +fresh-en-2,7,8,V,8,linear_attention,400,0,400,0.5560215732082725,1.2039144802838564,2.0314424350857734,1.2734395997726824,400,0.12547250613570213,0.3106498569250107,0.5931588768959046,0.33314794417470694,400,4.820614982297824,6.128494330842684,7.094957404951281,6.026502389966515,400,0.602576872787228,0.7660617913553355,0.8868696756189102,0.7533127987458144,400,0.01822307230110612,0.043625390655713345,0.0942202083191823,0.05117037549818112,400,0.18951182730331725,0.2371647744489458,0.3537573652581782,0.25748651524732225,400,0.36106797176007294,0.4341225222042292,0.5424168887721682,0.445176640232646,400,0.6394117038635599,0.7372037470835603,0.81168816496379,0.7310775114703125,400,0.0632972994900409,0.09417504359183901,0.14013009375203275,0.09834316311824985,400,6.0,6.0,7.0,6.39 +fresh-en-2,7,8,QKV,8,linear_attention,400,0,400,0.8897886473685503,1.4389472343027592,2.232282324135304,1.5217388226371258,400,0.1941849946975708,0.31244826316833496,0.5063274621963503,0.3405528075248003,400,5.909960235656925,6.612152601503761,7.118650009035148,6.55432970176772,400,0.7387450294571156,0.8265190751879701,0.8898312511293935,0.819291212720965,400,0.017686950399966022,0.02998483017147148,0.050521024781157074,0.032546556111570715,400,0.1897255665647159,0.2178013326265698,0.27631897953375517,0.22587070717356475,400,0.361817887432852,0.4006645456961199,0.4651564330240367,0.4073860225363804,400,0.6406827407529547,0.6955173933204172,0.749012257004246,0.6957762279234827,400,0.09207785183231391,0.11367357252237731,0.1433481835512107,0.11655922520866949,400,6.0,7.0,7.0,6.7775 +fresh-en-2,8,9,Q,16,linear_attention,400,0,400,0.7115360419731587,1.0439937710762024,1.3857736611738802,1.0443064531049457,400,0.09236290529370308,0.14059225469827652,0.22423778474330908,0.15928514771163463,400,10.329231396140152,12.50995647189253,13.753422460182032,12.13841180441078,400,0.6455769622587595,0.7818722794932831,0.859588903761377,0.7586507377756737,400,0.010889786745102558,0.018598754397082296,0.03660020219070337,0.02440184357454656,400,0.10994611336698411,0.13598514571417475,0.1933031960350892,0.14976845845028763,400,0.38287847598554936,0.43292014856463196,0.5154933732154057,0.44311698135985655,400,0.6578383762955569,0.7086164120112666,0.7677076496489134,0.7119455855672101,400,0.07573352171213578,0.107449156005896,0.1366202930602535,0.10643264475801736,400,12.0,13.0,13.0,12.64 +fresh-en-2,8,9,K,16,linear_attention,400,0,400,0.6807672576513142,0.974963310174644,1.2984531952999534,0.9798759545030771,400,0.09328625351190567,0.13503039628267288,0.2059700295329094,0.14318987669423222,400,10.599212184993458,12.667955982703834,13.891067934837128,12.378879961150792,400,0.6624507615620912,0.7917472489189896,0.8681917459273205,0.7736799975719245,400,0.010121289752088163,0.017535346976904887,0.03397217460359147,0.021134973098446478,400,0.10942133532498555,0.1391963540590544,0.18547684764023045,0.14790627644423307,400,0.38234734224822253,0.42822365600308504,0.5020354918323987,0.4364086050737439,400,0.6521138471737558,0.6943136286478239,0.7503942786056949,0.6984117187126725,400,0.08238852002209573,0.11823542816280877,0.14455765341816013,0.11489309234760356,400,12.0,13.0,14.0,12.845 +fresh-en-2,8,9,V,16,linear_attention,400,0,400,0.4831611223286018,0.7940475479699671,1.3342432292178272,0.8646432308707154,400,0.06950774565339089,0.1110759899020195,0.2090410217642784,0.12714619062840937,400,10.272105042495769,12.430409776684805,13.710094886495344,12.145875429365494,400,0.6420065651559855,0.7769006110428003,0.856880930405959,0.7591172143353434,400,0.011134949176327338,0.019144482466073233,0.03717444247163193,0.022881927123128996,400,0.1097338134388892,0.14129537357717215,0.19121580522678916,0.1488918302125328,400,0.38390648797839433,0.4366837985809124,0.5160729927614417,0.4443359596968593,400,0.6601182355654198,0.7059374978885744,0.7620222794863211,0.7096365849417546,400,0.07539219480736424,0.10731985245973782,0.13805845186798446,0.10680251068936006,400,12.0,13.0,13.0,12.635 +fresh-en-2,8,9,QKV,16,linear_attention,400,0,400,0.7645834688097238,1.021525981836021,1.2748212238773704,1.0192790886823786,400,0.09697603061795235,0.1280551478266716,0.18325358629226687,0.13993709091097117,400,11.107424645658568,13.168533920400463,14.203771042584014,12.789787453575755,400,0.6942140403536605,0.823033370025029,0.8877356901615009,0.7993617158484847,400,0.00843078246283047,0.014334507730421782,0.029365461584506727,0.019042905592287513,400,0.10368784826509282,0.12621936810909629,0.17411956204555842,0.13835261192646756,400,0.3641499531736103,0.40935860854645095,0.48851941828314227,0.4199359174776645,400,0.6396587235724812,0.6797688431514963,0.7452839228530924,0.6883678002534177,400,0.08084619378954223,0.12310327325881887,0.1498354664474026,0.119379236478552,400,12.0,13.0,14.0,12.92 +fresh-en-2,9,10,Q,16,linear_attention,400,0,400,0.5841197005007416,0.9401031583547592,1.4729133268818264,0.9903671794326511,400,0.07781185954809189,0.13904079049825668,0.2652950644493104,0.17575458633713426,400,8.834842438419205,12.49349293170403,14.012138705808312,11.782312319754995,400,0.5521776524012003,0.7808433082315018,0.8757586691130195,0.7363945199846872,400,0.00945783380741601,0.018711113818747217,0.054068444625634635,0.034537719667985134,400,0.10940713010122523,0.14094804669279787,0.2511805370281058,0.17006863473202954,400,0.3753042159680791,0.4394531111199811,0.5342787203728024,0.4539802792505677,400,0.648604145073041,0.7021013378739127,0.7742851065618969,0.7105254654141082,400,0.07752604852166226,0.10976186862783553,0.1395437819744613,0.10913923282606021,400,12.0,13.0,13.0,12.63 +fresh-en-2,9,10,K,16,linear_attention,400,0,400,0.6156108346767724,0.861966734752059,1.2304173525422812,0.8947859468188836,400,0.07210212349891662,0.1157320886850357,0.1794776350259781,0.12220309496857226,400,11.17057074991446,13.114997978301568,14.24247764727429,12.887838262200114,400,0.6981606718696538,0.819687373643848,0.8901548529546431,0.8054898913875072,400,0.008226670554726416,0.014665154393878506,0.0288223512483474,0.016937872218062325,400,0.10241423667129484,0.1284378685409256,0.17829718634563343,0.13687032016345868,400,0.3615746771455679,0.40800557286487094,0.47561984330350465,0.41474870243072537,400,0.6370494619733148,0.679463546304351,0.7393533882566252,0.6832193307346404,400,0.09515347412595569,0.12293742099214601,0.1470742924190727,0.12220169709481282,400,12.0,13.0,14.0,13.0075 +fresh-en-2,9,10,V,16,linear_attention,400,0,400,0.41676295064389707,0.6541136433370411,1.1477090947329998,0.7216081794322236,400,0.05224834531545639,0.0884949117898941,0.15900364518165588,0.09917690190020949,400,11.030330260303392,12.9876210359939,14.1534030556992,12.767455621218362,400,0.689395641268962,0.8117263147496188,0.8845876909812,0.7979659763261476,400,0.00869801466718456,0.015462821384912647,0.030036444452319472,0.017772697435181093,400,0.10577751807946256,0.13057452366330297,0.18282527047182515,0.13808400618176797,400,0.3663485393564446,0.41596230948055457,0.4825421954347549,0.4204635050595473,400,0.635758083038972,0.6874030391119894,0.7468218358665392,0.6895351184726354,400,0.08417754588432542,0.1168618598251707,0.14936358399080804,0.11774560372145527,400,12.0,13.0,14.0,12.9075 +fresh-en-2,9,10,QKV,16,linear_attention,400,0,400,0.6403672892600298,0.9104771688580513,1.2211272411979737,0.9242272769659757,400,0.07458450347185135,0.11616862937808037,0.18310137242078783,0.134665567856282,400,10.772418672648032,13.380891517420249,14.482153948894448,12.828896672718257,400,0.673276167040502,0.8363057198387656,0.905134621805903,0.801806042044891,400,0.006987202117806005,0.013048999080038901,0.032352025308508364,0.02051454900092564,400,0.0994989242761261,0.12471060054240976,0.19908812856916658,0.14293030111576446,400,0.3540012165816263,0.40184301910594145,0.48065173784775683,0.41348121852942255,400,0.6272533863571255,0.6733312325553102,0.7396207959319867,0.6791005129726806,400,0.09278804322953894,0.1260527955764632,0.15369825164569187,0.12467007454416365,400,12.0,13.0,14.0,13.0675 +fresh-en-2,10,11,Q,16,full_attention,200,0,200,0.6217361217597499,0.8588953739963472,1.0879042023792862,0.8613086555473274,200,0.08656363412737847,0.11629294976592064,0.159348963201046,0.12042190544307232,200,11.3953228520266,12.512731300762585,13.4132782416839,12.47613037042567,200,0.7122076782516625,0.7820457062976616,0.8383298901052437,0.7797581481516044,200,0.012856526402063846,0.018579958774202508,0.026938991010733104,0.019195891105872546,200,0.11513253343128332,0.13881364299826554,0.17143097159637222,0.14102982263703945,200,0.39080525786206677,0.4308645946931363,0.47394323563994545,0.4325617177089401,200,0.6635725741124127,0.7006620171496789,0.7435008072174482,0.7021341925663233,200,0.08176240805240446,0.11046676702898083,0.13614845093563396,0.10948950681123809,200,12.0,13.0,13.0,12.65 +fresh-en-2,10,11,K,16,full_attention,200,0,200,0.6621845734771341,0.8579957305919379,1.0429124694317577,0.8506134263938293,200,0.09501790776848792,0.17020879685878754,0.25952345728874204,0.17819154888391495,200,7.307132658341592,10.469349695381915,12.537312808304588,10.130102374869383,200,0.4566957911463495,0.6543343559613697,0.7835820505190367,0.6331313984293364,200,0.01841277878927808,0.035339641805613276,0.07932333009002468,0.043750259883175345,200,0.12773404649456147,0.20972890196376165,0.3066036272778399,0.20850696095522572,200,0.43167387097240234,0.49689485396721034,0.5804362123769544,0.5058431771795857,200,0.7057683465062268,0.7578576540823418,0.7957970965701755,0.7523645144562694,200,0.059384090110283,0.08781282492575032,0.119986607189019,0.08902220527905698,200,11.0,12.0,13.0,12.04 +fresh-en-2,10,11,V,16,full_attention,200,0,200,0.7910943925846368,0.97863654140383,1.2328994596377014,0.9979268654249609,200,0.09664468690752984,0.12546798586845398,0.19318059533834456,0.13570110753178596,200,11.792154294328554,12.78981509951631,13.738854911050506,12.782749446567596,200,0.7370096433955347,0.7993634437197694,0.8586784319406566,0.7989218404104748,200,0.010976020692748436,0.01673302741598156,0.023789421906500623,0.017062277585833733,200,0.11088898793367176,0.12723890935876303,0.16986177556046966,0.13482035420128782,200,0.38509818859334677,0.42274297853856635,0.45433023531308514,0.42177799257752263,200,0.6527571320931893,0.6909240875029913,0.7220980546231547,0.6933899255987148,200,0.09048084929841958,0.11973222303212816,0.1435584635712541,0.11529275691700139,200,12.0,13.0,13.099999999999994,12.8 +fresh-en-2,10,11,QKV,16,full_attention,200,0,200,0.7359494174830615,0.9326356733217835,1.145884907245636,0.9298488935478963,200,0.09351496696472168,0.14523176848888397,0.18735939413309097,0.14044940054416657,200,10.717571898623689,12.14954155138565,13.478252681728977,12.129532125072826,200,0.6698482436639805,0.7593463469616031,0.842390792608061,0.7580957578170516,200,0.012473213794830885,0.02112817180101991,0.03285864636076488,0.021954165683340247,200,0.11469601524654695,0.14721384419545386,0.19606222929122147,0.1517285537708352,200,0.39547905106435316,0.4473112548724165,0.48655219965937613,0.4446178318221039,200,0.6678676742594226,0.7140064867943439,0.7477165556982082,0.7120801330591198,200,0.07804972536981533,0.10655233864527767,0.13865592130473667,0.1069933877044463,200,12.0,13.0,13.0,12.665 +fresh-en-2,10,11,JS,16,full_attention,200,0,200,0.011709015141423152,0.023373060074845853,0.043107747244721384,0.027677225003011473,200,0.0032383885001763704,0.006865991745144129,0.021535840816795822,0.012058183691115118,200,2.617147312838424,5.815193020227113,8.254860065664781,5.656953591645987,200,0.1635717070524015,0.36344956376419457,0.5159287541040488,0.3535595994778742,200,0.06255034166302798,0.11676127799835936,0.3412924810466721,0.17408890896488177,200,0.20915395264079262,0.31149713702621673,0.5872563399457557,0.3647079617739476,200,0.5989962260400519,0.7132297614635023,0.847698837654311,0.7207813611467898,200,0.8365399310151895,0.8915830096437295,0.9519819496264191,0.8918119046965123,200,0.012226791247638183,0.03101575282505921,0.05147123162550377,0.03220087854244019,200,6.0,9.0,10.0,8.37 +fresh-en-2,10,11,sqrtJS,16,full_attention,200,0,200,0.35895347251789644,0.4935784398112446,0.6759456943720579,0.5069780787237687,200,0.05690678879618645,0.08286124840378761,0.14674851298332212,0.09638299074023962,200,7.737020182887723,11.242860968377068,12.887845618082206,10.77721058656071,200,0.4835637614304827,0.7026788105235667,0.8054903511301379,0.6735756616600443,200,0.016098762050105,0.028208400768005237,0.07119894107235233,0.03943889962605179,200,0.12761073696529185,0.16533935603053623,0.27651501951729623,0.1893869334868225,200,0.42097097663387156,0.4825174697427024,0.5681556709248554,0.49465358437463036,200,0.6907857172104901,0.7337084473227219,0.7925056622144059,0.7367745111306826,200,0.07294075788625516,0.10327805304595494,0.12654921058093285,0.10235385037661761,200,11.900000000000002,13.0,13.0,12.465 +fresh-en-2,11,12,Q,8,linear_attention,400,0,400,0.6585037915036083,1.0366052743047476,1.698347765207291,1.1118832486122847,400,0.16585994511842728,0.269122451543808,0.43194699883461,0.2857397244684398,400,5.766887318107972,6.378005840331815,6.852469356128504,6.321907947598501,400,0.7208609147634965,0.7972507300414768,0.856558669516063,0.7902384934498127,400,0.023923220807822908,0.03633008552499549,0.055318651316807625,0.03887859885409399,400,0.21862549407077378,0.2562246356982312,0.30286513784294067,0.25898151301622235,400,0.3876836843206582,0.42757348431976333,0.4813609725200964,0.4313953994975205,400,0.6551193536133514,0.69607563471175,0.7344531629696013,0.6942436689199457,400,0.09674992031996818,0.12030398602912987,0.14672931704266212,0.12076796230809421,400,6.0,7.0,7.0,6.85 +fresh-en-2,11,12,K,8,linear_attention,400,0,400,0.49835739079862834,0.7622699113562703,1.3046596046537162,0.8510331864375621,400,0.11305943652987481,0.18373078852891922,0.31755511164665223,0.20454091554507614,400,6.03529956471883,6.582391198098376,7.071376965194263,6.570515051296469,400,0.7544124455898538,0.822798899762297,0.8839221206492829,0.8213143814120586,400,0.018760209145151592,0.030766260270822285,0.04650504642563264,0.03174970339675074,400,0.2043056066508603,0.23715554416801127,0.28217950789867813,0.24020283982354568,400,0.37136537348366455,0.4104696007016865,0.4550114125341802,0.4128543082569678,400,0.637871213879049,0.6825733554181741,0.7160147989598735,0.6795736259618192,400,0.09710450856524641,0.126263593032124,0.15787816234856153,0.12643945319755479,400,6.0,7.0,7.0,6.8625 +fresh-en-2,11,12,V,8,linear_attention,400,0,400,0.3862847808748484,0.691340209916234,1.2980188675224782,0.7843743969663046,400,0.08824722766876221,0.17121456563472748,0.3284576952457428,0.19910946891643108,400,5.685854589832213,6.454718360893167,7.123990747555283,6.426016882038779,400,0.7107318237290267,0.8068397951116458,0.8904988434444103,0.8032521102548473,400,0.01756658559201633,0.03420048970005613,0.05814292239942435,0.03652053052395963,400,0.19868639168482918,0.2452863847769825,0.30573357075116564,0.2489749251396205,400,0.3672617946409735,0.4211076946593562,0.48303015310338276,0.42379699164288737,400,0.6323070484406675,0.6859270871107883,0.7393421987609231,0.6866842719410409,400,0.0879796133077167,0.12453906611247592,0.1577216280429513,0.1248050757446608,400,6.0,7.0,7.0,6.79 +fresh-en-2,11,12,QKV,8,linear_attention,400,0,400,0.5960858000442386,0.9248353596776724,1.3828169412910942,0.9651207965565846,400,0.14527591764926912,0.22725511342287064,0.3509412556886673,0.2391181750036776,400,6.063482297071264,6.539409334908655,6.95838795906213,6.520378251282575,400,0.757935287133908,0.8174261668635818,0.8697984948827663,0.8150472814103219,400,0.02138451755322308,0.03190743977307786,0.04562484202200335,0.032932086800386305,400,0.21624988597682096,0.24726052579291943,0.2805743548299556,0.24793255908994405,400,0.3804082681794955,0.4168007836452087,0.4570258765267173,0.4175042367290196,400,0.6434106658886266,0.6822706703654456,0.7172646386012852,0.6810418443972257,400,0.10341423148344439,0.1269471468572378,0.15234006823375013,0.12737356391466184,400,7.0,7.0,7.0,6.9275 +fresh-en-2,12,13,Q,16,linear_attention,400,0,400,0.5140243185684085,0.7920748987235129,1.1551996864378453,0.8140374513078131,400,0.06810176596045495,0.13050541281700134,0.251472294330597,0.14869230702053757,400,7.874273309005751,11.88183766573001,13.534692318513763,11.31794255617015,400,0.49214208181285946,0.7426148541081257,0.8459182699071102,0.7073714097606344,400,0.012143157628396328,0.023106402567392298,0.06879571274137361,0.03242616866654701,400,0.1164679432137447,0.15833257620357294,0.2773857082299796,0.17768206318013377,400,0.39273568915913487,0.45555855734480527,0.5680478718475891,0.4686920617749388,400,0.660196853867253,0.7149244874266525,0.7869323944033125,0.7203228783552319,400,0.06899433487709615,0.10371179677678083,0.13568015701791938,0.10345697955422561,400,12.0,13.0,13.0,12.52 +fresh-en-2,12,13,K,16,linear_attention,400,0,400,0.44422444636002184,0.6919434266164899,0.9595235832035541,0.7011545427032979,400,0.05618530847132206,0.09301932528614998,0.15045668482780455,0.10033432116732001,400,10.896132260288613,12.92931383462771,14.296623088959658,12.67545317468886,400,0.6810082662680383,0.8080821146642319,0.8935389430599786,0.7922158234180537,400,0.007943026983376742,0.015833201310398573,0.03122808843760217,0.018803924330804918,400,0.1054994942503602,0.13410669269866463,0.18984302195683495,0.1430864474045428,400,0.363691579874421,0.4145974071768297,0.480399693824184,0.421414109193494,400,0.6353106666018974,0.6837706148599079,0.7386731066289942,0.6855477444922334,400,0.0843145747951138,0.11882039731830371,0.15189258629770644,0.11832944097123402,400,12.0,13.0,14.0,12.885 +fresh-en-2,12,13,V,16,linear_attention,400,0,400,0.376494140806608,0.541369981598109,0.7962690744549036,0.5764037169746007,400,0.04890311174094677,0.08070170879364014,0.13778375983238222,0.0900493144756183,400,9.527226237796368,12.360909484199293,13.843344504417884,11.99486402921892,400,0.595451639862273,0.7725568427624558,0.8652090315261177,0.7496790018261825,400,0.010386004584463192,0.019626876421897258,0.04529328083330095,0.024739735408045672,400,0.11004525402605796,0.14458336260850846,0.20806728621438073,0.15546372474206735,400,0.3737145111206523,0.43525136714895185,0.5446724807047612,0.44695648296364354,400,0.6523517020951934,0.707901832186951,0.7778340035212765,0.7104704246967427,400,0.07093523604405214,0.10413444660212752,0.13966353078490817,0.10520553208159095,400,12.0,13.0,13.0,12.5425 +fresh-en-2,12,13,QKV,16,linear_attention,400,0,400,0.5387952663935721,0.7266457374207675,0.9496303943917156,0.7384376674389932,400,0.06586806327104569,0.10212142392992973,0.17609018981456756,0.11457111104391515,400,9.476262916606661,12.947377101791986,14.181008159236807,12.363944172275371,400,0.5922664322879163,0.8092110688619991,0.8863130099523004,0.7727465107672107,400,0.008551307445155348,0.01571810774166306,0.04589590617656452,0.02225628328954557,400,0.107075439997774,0.13595151685328521,0.23367813466074183,0.15390933474586632,400,0.3684700040285315,0.41702953683889665,0.5146765833114179,0.4303525675793141,400,0.6381493072555713,0.6832719126453037,0.7575018798145321,0.690981991446235,400,0.08155230895063056,0.11706748204150516,0.15319979313781287,0.11643783478021458,400,12.0,13.0,14.0,12.855 +fresh-en-2,13,14,Q,16,linear_attention,400,0,400,0.42049874875228854,0.6985972025431693,1.0954076325055213,0.7200234394970175,400,0.06011932790279389,0.10995443910360336,0.19416973143815996,0.12071008232887834,400,9.162027009890991,11.446607892785817,13.212946769971035,11.356921255088993,400,0.572626688118187,0.7154129932991136,0.8258091731231897,0.7098075784430621,400,0.014062248601453034,0.02651961574055583,0.04975589869505585,0.02934730008147822,400,0.12073815146540877,0.15511095905245254,0.21539201670479174,0.16448574644534125,400,0.3992550324521137,0.4702941416850507,0.5472397187304222,0.4716237269053883,400,0.6768506602356483,0.7412111068388272,0.8046870765854471,0.738592453266898,400,0.061167809322643624,0.09106666122986318,0.12516601039308387,0.09257473676133615,400,11.0,12.0,13.0,12.145 +fresh-en-2,13,14,K,16,linear_attention,400,0,400,0.4758277563843876,0.6827202637214214,1.057620588410646,0.7333224972593598,400,0.06510360017418862,0.11021097749471664,0.2081768810749054,0.12774660379625857,400,8.980181122005643,11.521050686512435,13.314499401001456,11.309624952967015,400,0.5612613201253527,0.7200656679070272,0.832156212562591,0.7068515595604384,400,0.013446529720084784,0.02591748653125247,0.052113450811571156,0.030320957863654064,400,0.12136523098862693,0.15721476075394644,0.2321326793025227,0.170089630661764,400,0.39901906867964754,0.46555782945250557,0.5501973891413354,0.4713077309972695,400,0.673487442721642,0.730033003203183,0.7944826314665893,0.7323775137275578,400,0.06511153296210259,0.09582224228560879,0.1289328561222271,0.09687990705826895,400,11.0,12.0,13.0,12.275 +fresh-en-2,13,14,V,16,linear_attention,400,0,400,0.30862216246314345,0.5312029414344579,1.0219840514473617,0.6212735211847757,400,0.045331576839089394,0.08282066881656647,0.20124808400869373,0.10813434640411287,400,8.992598925655413,11.350617520662826,13.248108879949134,11.201552822160334,400,0.5620374328534633,0.7094135950414266,0.8280068049968209,0.7000970513850209,400,0.013847991994298562,0.02730766245917689,0.05194945383731177,0.031540521598602086,400,0.12092832728121225,0.15668393060403474,0.23699532797560094,0.16993667231881587,400,0.4000641499410667,0.4737242153864769,0.550510154174634,0.47554504123131885,400,0.6743398373025041,0.7361090009961326,0.8028703638614197,0.7394394933665182,400,0.061502177138027525,0.08960915296860791,0.1258830479843296,0.09198020681054957,400,11.0,12.0,13.0,12.115 +fresh-en-2,13,14,QKV,16,linear_attention,400,0,400,0.48345836913213136,0.677657678257674,1.068379379063845,0.733251446299837,400,0.0650777868926525,0.10329722985625267,0.1931288555264473,0.11795299745164811,400,9.848221217927714,11.954553764669317,13.426058184609053,11.750584120806014,400,0.6155138261204821,0.7471596102918323,0.8391286365380658,0.7344115075503759,400,0.012780828555945102,0.02256014342544023,0.0416439350857866,0.025719569482384595,400,0.11656162320577884,0.15067448433241595,0.21081136887559945,0.1578423051113009,400,0.3940907516459466,0.45018347929992764,0.5230745381936387,0.45582092482299785,400,0.669903351195225,0.7207791397968842,0.7866715311365536,0.7245171354178033,400,0.06856908740434238,0.1011358195210721,0.12883521299310802,0.10015703404234326,400,11.0,13.0,13.0,12.4025 +fresh-en-2,14,15,Q,16,full_attention,200,0,200,0.7049972745124251,0.8899055793881416,1.1669517185539007,0.9203884942614241,200,0.1240799032151699,0.1511080414056778,0.186514076590538,0.1534906815737486,200,9.503825882184657,11.174141047488817,12.702964606742553,11.119277171337721,200,0.593989117636541,0.6983838154680511,0.7939352879214095,0.6949548232086076,200,0.017303233253231038,0.02879216623273185,0.04556924952720596,0.03038624733290586,200,0.1349017368789534,0.16770862522174174,0.21542470858880955,0.17112763128678515,200,0.43059907741352055,0.47592402079533874,0.5434344902613643,0.4803329820455383,200,0.6934577041475769,0.7381528979159269,0.7885428321222366,0.7401341938504029,200,0.061372071255080755,0.08446531177219777,0.11785570645136519,0.08705442820343615,200,11.0,12.0,13.0,12.055 +fresh-en-2,14,15,K,16,full_attention,200,0,200,0.45457539928611373,0.6414374079322442,0.8653815173543988,0.6461106917937286,200,0.07908274456858636,0.12040147557854652,0.22711606472730636,0.144260450899601,200,7.204369813443333,8.645827316830077,10.507961661652688,8.749846571517578,200,0.4502731133402083,0.5403642073018798,0.656747603853293,0.5468654107198486,200,0.03484372072810417,0.056709255645598214,0.08139181294403315,0.05838574212628363,200,0.15131892568507502,0.21290678939925223,0.27013355097804137,0.21577672181211788,200,0.5125191982669594,0.5870638359900611,0.6600453982861173,0.5871069600615655,200,0.7756898716995415,0.8133486899155924,0.8557175525727936,0.8165585936352225,200,0.03781294526171493,0.05061813563414566,0.07682514540786327,0.05352413251675745,200,10.0,11.0,12.0,10.68 +fresh-en-2,14,15,V,16,full_attention,200,0,200,0.4376588855870069,0.5745079589542001,0.8202009979984722,0.5979214567807503,200,0.07054382637143135,0.11756329610943794,0.201433789730072,0.13071663424372673,200,6.993813354137333,9.502488411628033,11.173564591082418,9.307959857131271,200,0.4371133346335833,0.593905525726752,0.6983477869426511,0.5817474910707044,200,0.02879716657027992,0.045587345657064605,0.08585920150173122,0.05179412105106945,200,0.16116937969705944,0.19552122153860346,0.2935391270012225,0.21266438212015878,200,0.48055182039679933,0.5499934886925004,0.6207462725634811,0.5523162002499324,200,0.725393864829569,0.7791946311405197,0.8346104175327279,0.783780711555217,200,0.044673435958950854,0.06134272989925878,0.09588651958489398,0.06685632775907234,200,10.0,11.0,12.0,11.28 +fresh-en-2,14,15,QKV,16,full_attention,200,0,200,0.6537230805493891,0.7794557483866811,0.911126397177577,0.7828541885322192,200,0.11116750985383987,0.12902627885341644,0.15805570930242538,0.13244120195508002,200,9.07384844548802,10.53057160834012,12.056999577608554,10.55645541397451,200,0.5671155278430012,0.6581607255212575,0.7535624736005346,0.6597784633734068,200,0.02180200952197641,0.03462583220526033,0.050887279333119116,0.03563068161830936,200,0.1369891791787797,0.16929981055153218,0.21023479356583566,0.17131985011908718,200,0.45246805026559067,0.511033573909291,0.5692540338094116,0.5111686588743951,200,0.7185958869888703,0.7621207160887895,0.8114361646969457,0.7644355925077724,200,0.053444759849850344,0.07224500930324185,0.10663638382072271,0.0755794682677014,200,11.0,12.0,13.0,11.7 +fresh-en-2,14,15,JS,16,full_attention,200,0,200,0.014904379848849203,0.026413629350372503,0.0540445632568435,0.03173870663458047,200,0.004110403498634696,0.0087392907589674,0.019521798379719256,0.011973647387931123,200,2.861618878531048,5.263653175745896,7.449828850302678,5.275311005872961,200,0.1788511799081905,0.3289783234841185,0.4656143031439174,0.32970693786706007,200,0.07651338448512326,0.13598132604987107,0.3060867089903157,0.16751286446273184,200,0.2222518855844168,0.3230173395912448,0.528403858435191,0.3538174925693752,200,0.63932631152739,0.7556958862192491,0.8842163670081871,0.7586556977394497,200,0.8747839717769333,0.9299194141226115,0.9748954353768851,0.9263114769021038,200,0.0043998137459033675,0.013607161124437913,0.029801979308480706,0.016334177442489106,200,5.0,7.0,9.0,7.1 +fresh-en-2,14,15,sqrtJS,16,full_attention,200,0,200,0.38702396722510457,0.506677265977487,0.707418407453224,0.5327011090918677,200,0.06411192044615746,0.09348417446017265,0.13972039222717286,0.10226278582587839,200,7.170381058306093,10.101540135856517,12.000093265242583,9.853055460709545,200,0.4481488161441308,0.6313462584910323,0.7500058290776614,0.6158159662943465,200,0.022221571063954997,0.038928103948196784,0.08209690865050921,0.046867267852027876,200,0.13829966249706946,0.1801738810065037,0.2613564666782947,0.19297507424713756,200,0.45189410439643896,0.5258846711689636,0.6351730220499668,0.5368699061987069,200,0.727661096728591,0.783550899996448,0.8467842429308771,0.7843328184845805,200,0.04680053111145946,0.07342024934251046,0.09930788474187263,0.07250668411934788,200,10.0,12.0,12.099999999999994,11.415 +fresh-en-2,15,16,Q,8,linear_attention,400,0,400,0.2320575937628746,0.5322947492823005,0.9188685007393361,0.559514098247746,400,0.0472423680126667,0.11171611398458481,0.2524071633815766,0.13249188045505433,400,6.092628934655876,6.9917212111703595,7.5293326445809345,6.8600262858100844,400,0.7615786168319845,0.8739651513962949,0.9411665805726168,0.8575032857262606,400,0.00893016740475101,0.020601516766076564,0.044723156428977937,0.025025582764470027,400,0.17497371085421026,0.21793800585465006,0.2865698437033712,0.2267381196699099,400,0.331596422282716,0.38473894855222435,0.4587044440794619,0.39070012922871017,400,0.6003033099244138,0.6480833171909904,0.6998345051171003,0.6499021649704855,400,0.1194707958933655,0.14829763454647954,0.17762547775044726,0.14859746028409979,400,7.0,7.0,7.0,6.9825 +fresh-en-2,15,16,K,8,linear_attention,400,0,400,0.34262026101350784,0.4971428979188204,0.7807130020111799,0.5310543482680805,400,0.06791906505823135,0.09871680662035942,0.17410916537046434,0.11112485434859991,400,6.518257990587267,7.143171478334613,7.50875932857435,7.062172275708563,400,0.8147822488234083,0.8928964347918267,0.9385949160717938,0.8827715344635704,400,0.009346055173906798,0.017135851433285092,0.03247454087671984,0.019491178117720123,400,0.17530283539612135,0.20274180443933637,0.24309324932461412,0.2063879982120222,400,0.33364534112453303,0.3716677462837455,0.4277645887028576,0.377350375956223,400,0.6017706266195831,0.6417687834736032,0.6821793213838873,0.6431848589515745,400,0.12444789018833036,0.15167583006297886,0.17584070087276774,0.15103809420178305,400,7.0,7.0,7.0,6.9925 +fresh-en-2,15,16,V,8,linear_attention,400,0,400,0.32602587770670655,0.4621168877929449,0.7086701847612858,0.49040348157519476,400,0.06605661436915397,0.09446703270077705,0.16306437849998476,0.10618776489049196,400,6.263861332508568,6.992001949390662,7.47726908207445,6.907336182267288,400,0.782982666563571,0.8740002436738328,0.9346586352593063,0.863417022783411,400,0.009987057034498651,0.020594920497567996,0.03959538698888185,0.0234216272444666,400,0.17644309316809192,0.21015049973555658,0.2636617949382394,0.2158625973810412,400,0.33610390574607335,0.3806103899526042,0.4435405500634746,0.3867185703502098,400,0.607336420564203,0.6523536741265021,0.7024837519510545,0.6543994685092415,400,0.10984429635563063,0.1420636937652275,0.17200168756686693,0.14167503409984736,400,7.0,7.0,7.0,6.965 +fresh-en-2,15,16,QKV,8,linear_attention,400,0,400,0.37200659178197387,0.5081001911312342,0.7878896970301866,0.5486908619687892,400,0.0702376864850521,0.10316241160035133,0.18526293486356737,0.11697604967281222,400,6.563621919816764,7.1215012116356835,7.511731622909114,7.080342978912206,400,0.8204527399770954,0.8901876514544604,0.9389664528636392,0.8850428723640258,400,0.009285830694326213,0.017622664738356546,0.031262753237405336,0.018961252042295266,400,0.1760623462927111,0.20970139901764695,0.24491636633738073,0.20956979455483943,400,0.3360903035758347,0.3763538170686719,0.4228447583942967,0.3765759853725727,400,0.6017050275108086,0.6392522680250385,0.6790912422993212,0.6395308363905688,400,0.12947080538561662,0.15434068220642166,0.17772254938263024,0.15408179181000842,400,7.0,7.0,7.0,7.0 +fresh-en-2,16,17,Q,16,linear_attention,400,0,400,0.1085345599334687,0.3044695033458993,0.5816921635996553,0.3317287538452365,400,0.01683701928704977,0.049399444833397865,0.11030996143817902,0.058014235245063904,400,9.229399787565718,11.886896362316751,13.234198517401213,11.516750571159532,400,0.5768374867228574,0.7429310226447969,0.8271374073375758,0.7197969106974708,400,0.013932597831426786,0.023068006376980647,0.04890615759698003,0.028173328629623212,400,0.12057433153286591,0.15549514767286884,0.23648735264615434,0.16957008243780244,400,0.403095806095696,0.45259752386454255,0.5251389796537858,0.4608624075511165,400,0.6753789030552525,0.7160758468890634,0.7680216261011761,0.7186141916510397,400,0.07696109625121009,0.10065438455511688,0.1241151191308039,0.10036718667623962,400,12.0,13.0,13.0,12.4725 +fresh-en-2,16,17,K,16,linear_attention,400,0,400,0.33116912653204056,0.46913142944686115,0.6506060735322535,0.4812123993656132,400,0.04388701356947422,0.0646263025701046,0.09916348606348042,0.0697169864596799,400,10.87705579406867,12.457501568607814,13.464126368375455,12.270798161065265,400,0.6798159871292919,0.7785938480379884,0.8415078980234659,0.7669248850665791,400,0.012556199623217088,0.018957782145829757,0.031399088282413314,0.02108005474020307,400,0.1145285039390494,0.13915957265075468,0.17745788813584668,0.14421377076112074,400,0.3928316145957058,0.4363479765460262,0.490318864802938,0.44091555492957424,400,0.6700197563849883,0.7034869357897249,0.7515198644850047,0.7073033561722883,400,0.08374345367302802,0.10508455141487005,0.12361803168150694,0.10423911605414356,400,12.0,13.0,13.0,12.585 +fresh-en-2,16,17,V,16,linear_attention,400,0,400,0.28255533720366655,0.423395493067801,0.6171590889338404,0.4434045745941694,400,0.03979641795158386,0.06506848335266113,0.10764199793338779,0.07013003543019294,400,10.09951054613533,11.958473006345713,13.328806294171443,11.820632508603271,400,0.6312194091334581,0.7474045628966071,0.8330503933857152,0.7387895317877045,400,0.013360505256380906,0.022530927727687042,0.03894903736665824,0.02482202165340298,400,0.11826739633807055,0.14877456166822284,0.20391101382022597,0.15686592937223254,400,0.39400684420372545,0.45158922151833314,0.513464475044495,0.4533322668834755,400,0.6707049864585859,0.7164318789297386,0.7706257095443808,0.7183819383351073,400,0.07070406254588904,0.0964823347720808,0.12163351131454482,0.0971764623563012,400,12.0,12.0,13.0,12.345 +fresh-en-2,16,17,QKV,16,linear_attention,400,0,400,0.32350265779532494,0.42999072023667395,0.5844419991131872,0.4470250720006879,400,0.04192870482802391,0.062201663851737976,0.094621791690588,0.06564131278079004,400,11.124814148890323,12.440573657442897,13.514234330125452,12.349607539070137,400,0.6953008843056452,0.7775358535901811,0.8446396456328408,0.7718504711918835,400,0.012262465718847473,0.019074341063877013,0.02921511806282596,0.02036726513200726,400,0.1145015379225922,0.14024794291861875,0.18191238830038425,0.14600216013468897,400,0.3919264023828508,0.4331876597144522,0.47996719785151387,0.43598256306971395,400,0.6669100879930713,0.6986729381404595,0.7363915671355837,0.701304738617103,400,0.08882192647304224,0.10731304228675474,0.12782467412950016,0.10747044825949968,400,12.0,13.0,13.0,12.6875 +fresh-en-2,17,18,Q,16,linear_attention,400,0,400,0.13763023200444877,0.3529496125411242,0.5568669612053784,0.35054396654159065,400,0.030687961168587206,0.0800715796649456,0.13235853016376498,0.08292097771074623,400,6.658014901675,9.243927936934593,11.321158567131214,9.065815690553787,400,0.4161259313546875,0.5777454960584121,0.7075724104457009,0.5666134806596117,400,0.027552196919900872,0.04872440847686482,0.09354124706469702,0.05642254382577788,400,0.17142501418176423,0.22711146033201887,0.3262057494171524,0.2415134645887589,400,0.4663069913505754,0.5318772971829004,0.613014221545787,0.536602171746518,400,0.7105110959780384,0.7602443471740941,0.8198411769674526,0.7629732328056466,400,0.05998198612638311,0.0889009804126683,0.11650302900768214,0.08845847442436043,400,11.0,12.0,13.0,12.0125 +fresh-en-2,17,18,K,16,linear_attention,400,0,400,0.19883154276758433,0.2954140636138618,0.40346070094965397,0.29965475216333287,400,0.04514831975102425,0.0642731711268425,0.09372542873024942,0.06783392469864338,400,7.906841109318626,9.591328927092752,11.05916798171566,9.513039838876228,400,0.49417756933241413,0.599458057943297,0.6911979988572288,0.5945649899297643,400,0.029784235439758857,0.044544895554277306,0.06823767220344004,0.047517636753710076,400,0.18330129880890875,0.22452382334950755,0.2783122954801211,0.2281208903763998,400,0.46882541920942256,0.5182381573417467,0.5724095001831397,0.519114885578425,400,0.7086208819167649,0.7491074952133822,0.7917617115412752,0.7491364760497023,400,0.07306811501924196,0.09330699933777993,0.11448166563062899,0.0936800138964556,400,12.0,12.0,13.0,12.2525 +fresh-en-2,17,18,V,16,linear_attention,400,0,400,0.28444457799196243,0.3630967023782432,0.45597007940523326,0.3637144808400626,400,0.0793171465396881,0.1099793016910553,0.14340544492006302,0.11055116172879935,400,5.449002454264216,7.169715087298332,9.159322458861219,7.292064867678073,400,0.3405626533915135,0.44810719295614576,0.5724576536788262,0.4557540542298796,400,0.04979031588999661,0.08210726814932501,0.12908788777952088,0.08534510477266291,400,0.2275076766560479,0.304358772877504,0.3852397863253127,0.30553109295196634,400,0.5202239070530834,0.5809983232691766,0.6408350362306928,0.5804667233276897,400,0.7446693264412652,0.7903827221443913,0.8341884896091187,0.7887792027778626,400,0.05523496896452342,0.07734455379774244,0.1007349571650742,0.077462259151827,400,11.0,12.0,13.0,11.6325 +fresh-en-2,17,18,QKV,16,linear_attention,400,0,400,0.2726540342206135,0.3497917263302952,0.4507372446823866,0.35851804855599767,400,0.06908265724778176,0.09028414636850357,0.11531217396259309,0.0914928696397692,400,6.99736486634175,8.626527176708391,10.199950754906038,8.621321676547014,400,0.43733530414635935,0.5391579485442745,0.6374969221816273,0.5388326047841884,400,0.037909002368310844,0.05698295606792783,0.08577167323521356,0.05969194416241507,400,0.20867346104384704,0.25352088080773016,0.318953709083286,0.2581033836577386,400,0.4847197183754404,0.5421503847934435,0.596555002856954,0.5405944285523794,400,0.7239841826850536,0.7607130600760493,0.8091307269586864,0.7625232584955556,400,0.06783719863765901,0.08910564246661917,0.10996562019324782,0.08922603927521826,400,11.0,12.0,13.0,12.08 +fresh-en-2,18,19,Q,16,full_attention,200,0,200,0.42757751084864143,0.5230707763694227,0.623913636431098,0.523064110004343,200,0.05711409337818623,0.07461196184158325,0.09367594867944717,0.07470580890774726,200,11.466939779549499,12.33142999800727,13.481048622419559,12.43487338785972,200,0.7166837362218437,0.7707143748754544,0.8425655389012224,0.7771795867412326,200,0.012456870719173582,0.019833247767654966,0.02635437368042571,0.019425617263753484,200,0.11472723122608172,0.14210603154735793,0.17203850183889974,0.1431110726478567,200,0.39747848194073016,0.4376963473520394,0.4673599444431738,0.4334353424413451,200,0.6624763710759516,0.7010310731547114,0.7276196839336242,0.698137449359788,200,0.08707615862431772,0.10363501618374782,0.13017170704975234,0.10571968263476972,200,12.0,13.0,13.0,12.61 +fresh-en-2,18,19,K,16,full_attention,200,0,200,0.20267673855414617,0.26819515600800514,0.38065635652746993,0.2836863338272087,200,0.030279337987303735,0.04774639382958412,0.07978882864117622,0.05156202740967274,200,6.9299383809995305,11.589019573828669,12.560871745914906,10.804585159062722,200,0.43312114881247066,0.7243137233642918,0.7850544841196816,0.6752865724414201,200,0.0182534377753106,0.025375051020589817,0.08725504007676868,0.03835028287723017,200,0.12665546188124854,0.15545078681979718,0.32251684503747546,0.18796522027918108,200,0.42949668474000813,0.4730632412117409,0.5794746658855122,0.48465156572000195,200,0.6993039538116211,0.734882659380923,0.7727455707432611,0.7347246414117103,200,0.0715860178744928,0.09003949689377036,0.1137184324089872,0.09046827534980585,200,12.0,12.0,13.0,12.2 +fresh-en-2,18,19,V,16,full_attention,200,0,200,0.20082818543305622,0.29495606874115765,0.6489377400837838,0.34538337372941896,200,0.030973949842154982,0.04259492643177509,0.08486507013440128,0.04948415517807007,200,11.270526070956597,12.5153241698754,13.15626103425461,12.304607497077141,200,0.7044078794347873,0.7822077606172125,0.8222663146409132,0.7690379685673213,200,0.01441057002181766,0.01856220547063987,0.027984429958809228,0.02051271267254148,200,0.11948379833520846,0.1426770442590663,0.18009878482584504,0.14739157076583984,200,0.40052369223218276,0.4364863712970779,0.49386150582910465,0.44035768244442025,200,0.6644330382050988,0.6972576949266994,0.7391482885838664,0.6989389518054057,200,0.08870273412087466,0.10777155010632213,0.13419630339558541,0.11004420455784586,200,12.0,13.0,13.0,12.72 +fresh-en-2,18,19,QKV,16,full_attention,200,0,200,0.3187912561465055,0.3973900239216164,0.5418821140192449,0.4125086717179511,200,0.04147509410977364,0.05560419522225857,0.07609003633260726,0.05739068113267422,200,11.783751877479908,12.551006713767334,13.390163316749993,12.594410992552143,200,0.7364844923424942,0.7844379196104584,0.8368852072968745,0.787150687034509,200,0.012993802096629491,0.018319966636286975,0.023853459206771628,0.018279396113407467,200,0.1188703471968148,0.1370972023157161,0.16924785426310193,0.1398349652228136,200,0.3940755289456063,0.429976479939846,0.4579784135669662,0.425522191419923,200,0.6661560316585446,0.6937954723285793,0.7228878057773872,0.6928534788094453,200,0.08730105550587129,0.10398373882169466,0.1271444609130742,0.10548742279737996,200,12.0,13.0,13.0,12.61 +fresh-en-2,18,19,JS,16,full_attention,200,0,200,0.008706798572291063,0.015453939310873466,0.03442741068347459,0.01911709053846181,200,0.0037327888188883667,0.009708748199045658,0.0266639232635498,0.012770538186305202,200,1.5059478175314827,2.3283594286352995,4.116339641849155,2.6889541231606904,200,0.09412173859571767,0.14552246428970622,0.2572712276155722,0.16805963269754315,200,0.19252173614791507,0.3914527550231741,0.6416584119354828,0.3951190433640895,200,0.3956409484070341,0.6260428077448426,0.8054351139186814,0.6035878985229525,200,0.7962501512945249,0.8953137420203126,0.9632413753752798,0.8845474427626331,200,0.9318388328149693,0.9669840990752945,0.9886394396485646,0.9622941005012189,200,0.0023902814363606756,0.0070997641679930515,0.015701510673880717,0.00834999646838886,200,2.0,5.0,7.0,4.69 +fresh-en-2,18,19,sqrtJS,16,full_attention,200,0,200,0.2726464945706539,0.3351541784359142,0.4111551860230975,0.33943474419385894,200,0.06109372414648533,0.09853297099471092,0.16328971683979032,0.10558945152908564,200,4.581190917138312,6.8349791647576525,9.555185215208226,7.015182438322473,200,0.2863244323211445,0.4271861977973533,0.5971990759505141,0.43844890239515455,200,0.04496617828490613,0.08939369954827804,0.16617676051723185,0.09804455836945618,200,0.2054372444484531,0.3007280504621388,0.42127846114557815,0.30452780939168195,200,0.5441094234248772,0.6236289718739731,0.7210506092576822,0.624288171046703,200,0.7732819012082369,0.8172121234100993,0.8676815183336485,0.8195356628470304,200,0.04113805877573698,0.05931367226440325,0.0796330337137415,0.060107127351147156,200,10.0,11.0,12.0,10.84 +fresh-en-2,19,20,Q,8,linear_attention,400,0,400,0.19636926748789849,0.3737413245253265,0.7455098735168579,0.4317249461938627,400,0.0561393216252327,0.11652319878339767,0.2691425144672395,0.14500154570676385,400,3.781425561090029,5.3601973546721355,6.5043925661095265,5.285367241796815,400,0.47267819513625364,0.6700246693340169,0.8130490707636908,0.6606709052246019,400,0.032848339172776526,0.07035480165704709,0.15937204783001485,0.082338023266222,400,0.23558289767644666,0.3080199574311538,0.43976827110062566,0.32248501975995725,400,0.41137546752474796,0.49757233928735234,0.6189178983533311,0.507225095286354,400,0.6986783501620449,0.7781729574854495,0.8393759392324694,0.7729004649541511,400,0.04611218775255068,0.07977457802361287,0.12533499807559303,0.08310780628550236,400,5.0,6.0,7.0,6.1575 +fresh-en-2,19,20,K,8,linear_attention,400,0,400,0.2715590732172132,0.3915023892186582,0.5892058109864593,0.41847740480676293,400,0.06700540557503701,0.13213234394788742,0.21548925787210468,0.13812848344445228,400,3.9644857074188375,5.237713909518571,6.379980753076168,5.185816127496865,400,0.4955607134273547,0.6547142386898214,0.797497594134521,0.6482270159371081,400,0.036274632940844334,0.07534056217791668,0.14541673298290295,0.0849890932502861,400,0.2211288538878345,0.31495851303457045,0.43831249732726235,0.32375402667934283,400,0.41497044143403233,0.5085400588304572,0.603827091783469,0.5096543621422766,400,0.7146781850739098,0.793135609397762,0.8452458740866686,0.7871139775394203,400,0.052284735010082274,0.07615704305699926,0.11636898154706803,0.08089804937152166,400,5.0,6.0,7.0,6.08 +fresh-en-2,19,20,V,8,linear_attention,400,0,400,0.2860807956662029,0.3882336001843214,0.5453923731110991,0.4017216757388087,400,0.07818588316440582,0.11009020358324051,0.1486593157052994,0.11164668465964496,400,4.679217592501005,5.462776386624757,6.442537551489562,5.508469296504302,400,0.5849021990626256,0.6828470483280946,0.8053171939361953,0.6885586620630377,400,0.03453524964039636,0.06635097192331006,0.10138396037154597,0.0677175304641024,400,0.2205279695088007,0.27653192198698834,0.35001018067429557,0.2832949663385313,400,0.4159615513000868,0.49343059615356283,0.564648585632793,0.4910596547834028,400,0.7143338810062791,0.7825154308458101,0.8369935778249042,0.7766638077569504,400,0.053039416667978005,0.07703381392118297,0.11406830664335589,0.08069623203013988,400,6.0,6.0,7.0,6.145 +fresh-en-2,19,20,QKV,8,linear_attention,400,0,400,0.31194027569144966,0.42065423633903265,0.5931314354762435,0.44246456083026714,400,0.08693632557988167,0.12472729757428169,0.2035938858985901,0.13940269940532743,400,4.334423673019394,5.407631796753555,6.391932861852066,5.379497786397486,400,0.5418029591274243,0.6759539745941944,0.7989916077315082,0.6724372232996858,400,0.03593967170483023,0.06848439796484443,0.12081438677746437,0.07498650116350601,400,0.2384753246246652,0.3019447968162412,0.39957510590191697,0.30958533981383385,400,0.4246707545636565,0.4912165863537561,0.5835972253436295,0.5000045410854458,400,0.7149501127286828,0.7797110573051228,0.8321428663735164,0.7773511702333551,400,0.053740398757915045,0.07950203808466441,0.11619507676837203,0.08234057986134341,400,6.0,6.0,7.0,6.1775 +fresh-en-2,20,21,Q,16,linear_attention,400,0,400,0.0944188019959256,0.1680199622642249,0.46999255025293685,0.22138458650631945,400,0.014066648390144109,0.025365962646901608,0.07215343192219735,0.03527150228386745,400,9.94542153212465,12.332537448822226,13.834321524744075,12.054440117128793,400,0.6215888457577906,0.7707835905513891,0.8646450952965047,0.7534025073205496,400,0.01043627378188883,0.01982540165800768,0.04058536642190044,0.023772514248491125,400,0.11415238065793695,0.1449168858870434,0.20887553905260986,0.15565118911152287,400,0.3784897791639427,0.43984841018365184,0.520770652627754,0.4469525918593737,400,0.6491178740153054,0.6989797713652504,0.7700941628948379,0.7045989875848571,400,0.08196609391826343,0.11121721888997373,0.135585486809666,0.11011050524863669,400,12.0,13.0,13.0,12.6825 +fresh-en-2,20,21,K,16,linear_attention,400,0,400,0.1553449919214472,0.3284362788544968,0.46580453640781344,0.3255010169596062,400,0.019309617951512335,0.04998108930885792,0.07802436202764514,0.051652491395361724,400,10.112869596649878,12.140647900375713,13.555393678617923,12.014119017021844,400,0.6320543497906174,0.7587904937734821,0.8472121049136202,0.7508824385638653,400,0.012022797635518475,0.02119245906767083,0.03880949552275217,0.023374987505603538,400,0.11746402116903758,0.14708840406711504,0.1997164962097928,0.1538883251724416,400,0.3946804893592625,0.4507209224256568,0.5191507282247122,0.45227312175362405,400,0.6591835646087836,0.7053762308186753,0.7572995643437069,0.706362779427598,400,0.08826001029498089,0.10980113125194035,0.12963658655305632,0.10953078095266065,400,12.0,13.0,13.0,12.7125 +fresh-en-2,20,21,V,16,linear_attention,400,0,400,0.24610960150603206,0.33345380052924156,0.4636389989871532,0.34665939436585175,400,0.036685409024357796,0.05327794328331947,0.07877509221434596,0.0563099325960502,400,9.650735735217621,11.961758330819446,13.591701403209575,11.774095242182323,400,0.6031709834511013,0.7476098956762154,0.8494813377005984,0.7358809526363952,400,0.011812593219218053,0.022506431471185556,0.04386037725062987,0.02540942495557738,400,0.11711459138056532,0.15849506971870725,0.22093748543810188,0.16407553907762198,400,0.39415425732103515,0.45199717380316057,0.5252389259368915,0.4552764471878465,400,0.6606170728395946,0.7035443625468585,0.7654001386290459,0.7079187180077756,400,0.08322719339825481,0.11206296650910452,0.1340001316373146,0.10957291798143194,400,12.0,13.0,13.0,12.6925 +fresh-en-2,20,21,QKV,16,linear_attention,400,0,400,0.2192056284286082,0.3048442618455738,0.43132772911340006,0.3180543637496885,400,0.031235205940902233,0.04505884647369385,0.0744130350649357,0.04928936725482345,400,10.54853366147434,12.336939471102387,13.434625556564349,12.12591724181429,400,0.6592833538421462,0.7710587169438992,0.8396640972852718,0.7578698276133932,400,0.012730165462966956,0.019794543040218077,0.034453411946210745,0.022181090789244334,400,0.1196964550855682,0.147663426437664,0.19485974580928841,0.153904185983046,400,0.3977922079599433,0.44243689923627283,0.5042185415706971,0.4464144991842383,400,0.659661631623256,0.6982276061626742,0.7470673451632667,0.701594577851577,400,0.09097804809875128,0.11217887654075043,0.1320368256478356,0.11200223589373332,400,12.0,13.0,13.0,12.7825 +fresh-en-2,21,22,Q,16,linear_attention,400,0,400,0.07465632332023234,0.1763467611745,0.3216036592843012,0.1842719748154923,400,0.00944373616948724,0.02016234863549471,0.04492337629199029,0.02447875889018178,400,11.489974358685906,13.529115638552211,14.41965328360699,13.176239360514693,400,0.7181233974178691,0.8455697274095132,0.9012283302254369,0.8235149600321683,400,0.007306449433027784,0.012175655442994992,0.026167930132232127,0.015299707206320883,400,0.09850261001602743,0.11986406603030042,0.16879798242503982,0.1288752609688239,400,0.3564816515023045,0.3966787826978869,0.4716702216214862,0.407213556426683,400,0.6328487142250238,0.668187681932958,0.7269915450220852,0.6743946039483469,400,0.09442408962594716,0.1262257020568354,0.15097898995177586,0.12492133267363645,400,12.0,13.0,14.0,13.04 +fresh-en-2,21,22,K,16,linear_attention,400,0,400,0.08607032083673402,0.23734343308024108,0.3494748550467193,0.23013745411095443,400,0.009764208178967237,0.02878650091588497,0.044984512403607374,0.028351587974466384,400,12.306782893026407,13.69663516776928,14.384490019313473,13.50020783840073,400,0.7691739308141504,0.85603969798558,0.8990306262070921,0.8437629899000456,400,0.007487278337847642,0.011211372182066085,0.020006431855014405,0.012703791839631391,400,0.09973423736048238,0.11633916168143431,0.1514512248690472,0.12222266447256452,400,0.35975750019190544,0.39224237876476253,0.44387400091772616,0.3964694375989395,400,0.6342348600367494,0.6641490677655085,0.702789232772759,0.666704540574569,400,0.10663006013771607,0.13163995043804105,0.15472944548814763,0.1312462945499237,400,13.0,13.0,14.0,13.215 +fresh-en-2,21,22,V,16,linear_attention,400,0,400,0.15097150593064726,0.2187712574377656,0.33062250157818207,0.23200085987729835,400,0.017389082722365858,0.026713361032307148,0.044422946870327,0.02900975369149819,400,12.123516251089276,13.443493024263015,14.24772073244708,13.28422429438394,400,0.7577197656930797,0.8402183140164384,0.8904825457779425,0.8302640183989962,400,0.008199111451688336,0.012677791288831078,0.021316724204018932,0.01402975999582461,400,0.10064683133587765,0.11903770139027724,0.15727264642600053,0.12513866998148138,400,0.36375490990386006,0.3964178184364736,0.44596621778918694,0.4018450259454208,400,0.6404647585238622,0.6741458008304297,0.7094336916910313,0.6748553358775312,400,0.09421735261008773,0.12281289406550458,0.1455637953736346,0.12207746365194243,400,12.0,13.0,14.0,12.9775 +fresh-en-2,21,22,QKV,16,linear_attention,400,0,400,0.1377473527798429,0.22763161081820726,0.32312043346464636,0.22664083327341358,400,0.015442042518407107,0.025853493250906467,0.04127332791686058,0.027464078390039504,400,12.578656784241392,13.779425358663643,14.399339272468593,13.571881802743754,400,0.786166049015087,0.8612140849164777,0.8999587045292871,0.8482426126714846,400,0.007410816896254725,0.010743431989218625,0.01813311245291724,0.012353065752644533,400,0.0991115348925056,0.11310238370270181,0.14767937554256752,0.11974534817076432,400,0.3574570285066608,0.38745698551623375,0.434436467421694,0.3931747333994167,400,0.6314371797980757,0.6616147261190861,0.7001875144950055,0.6635699854491495,400,0.10619212825382134,0.1306143631814062,0.15330991255189336,0.13017780839078574,400,13.0,13.0,14.0,13.195 +fresh-en-2,22,23,Q,16,full_attention,200,0,200,0.4077480868436396,0.5050136065110564,0.6341044765431434,0.5159503794205375,200,0.0643241509795189,0.09346378222107887,0.14473298937082288,0.10397742537781596,200,8.361438404634018,10.518244704039269,12.30221077821351,10.401023852394692,200,0.5225899002896262,0.6573902940024543,0.7688881736383444,0.6500639907746683,200,0.020038711503719456,0.034744588510674135,0.06090474549722623,0.038727804176916285,200,0.14226534576937322,0.18904798433998926,0.26578619389687175,0.19816989772034468,200,0.43839015202788456,0.4945601244139046,0.5553302326674202,0.4969300791234712,200,0.7050979452291856,0.7439753273532166,0.7826048305770043,0.7433319831584491,200,0.07203942007137484,0.09260553752437467,0.11093091531183412,0.09204702738079593,200,11.0,12.0,13.0,12.275 +fresh-en-2,22,23,K,16,full_attention,200,0,200,0.3755889473017306,0.47076327656395733,0.581899765599519,0.4729545893985778,200,0.08717131987214091,0.13724636286497116,0.22178117930889124,0.14249079771339893,200,5.435378124647292,6.934805246347662,10.194222073562033,7.568671746522788,200,0.3397111327904557,0.4334253278967289,0.637138879597627,0.47304198415767423,200,0.03796845500268491,0.08714716652241361,0.12960034093723702,0.08296236858822484,200,0.1893935052342641,0.3168746647082327,0.3869430771756581,0.29637058430905894,200,0.5063040539163814,0.5830770944622221,0.6325416870627971,0.5711063411653369,200,0.7447132827350567,0.7919737712246697,0.8155735258190694,0.7853247213264524,200,0.05935570799699881,0.07279473383185983,0.08710182772121657,0.07311338543335372,200,11.0,12.0,12.0,11.52 +fresh-en-2,22,23,V,16,full_attention,200,0,200,0.23842049092054368,0.31944643915630877,0.5010916820261624,0.33829028678825124,200,0.04843153841793537,0.10620660334825516,0.23915957659482945,0.12683624900877477,200,3.3867849355124995,6.319961977706251,9.887261323639493,6.481598718751385,200,0.21167405846953122,0.3949976236066407,0.6179538327274683,0.40509991992196154,200,0.04121874843574775,0.10212129352471129,0.24833903577570068,0.1260952975866286,200,0.18909270740037554,0.3281606458752409,0.5219596541903744,0.34333320647224697,200,0.5287464002532282,0.6194831380739054,0.7541563252536652,0.6364672949888174,200,0.7859215117997448,0.8312104503023139,0.8866019380897033,0.8345750326931796,200,0.03668195491471114,0.052776126478365964,0.0787661586026387,0.05470767903080984,200,9.0,10.5,12.0,10.4 +fresh-en-2,22,23,QKV,16,full_attention,200,0,200,0.36500207604840396,0.463483129395172,0.591547960229218,0.46953482207958586,200,0.0683653011918068,0.1145361103117466,0.19546185880899422,0.12639000492170452,200,5.68450203936997,8.349205648359964,11.13561512926826,8.462177238303555,200,0.3552813774606231,0.5218253530224978,0.6959759455792662,0.5288860773939722,200,0.02912209851682172,0.06109168875431771,0.12097804262640702,0.06723230822763684,200,0.16227343022583332,0.26477958399582235,0.3677970639682868,0.2626599213938303,200,0.48099272016296724,0.554040017768743,0.6180777985127296,0.550899471825888,200,0.735756746659926,0.7769178279885305,0.8103034565834726,0.7748067273259538,200,0.062487366454123124,0.07966180443947153,0.09900878833797638,0.0796716360105363,200,11.0,12.0,12.0,11.785 +fresh-en-2,22,23,JS,16,full_attention,200,0,200,0.0012135659423137214,0.007214870226903258,0.012908981638338446,0.007276498641497913,200,0.0005508021858986467,0.003033301909454167,0.00825609192252159,0.004046378126477066,200,1.9597116099469432,2.893045380434243,4.568850724774657,3.13245915280203,200,0.12248197562168395,0.1808153362771402,0.28555317029841604,0.19577869705012688,200,0.16679896723177956,0.30204067509107413,0.4776355939223207,0.31428818102730444,200,0.3570597728925854,0.5292406933011643,0.7051504978165276,0.5258299567516221,200,0.7918961961439432,0.8734844680449818,0.9327463827750408,0.8671854212585104,200,0.9305979376284169,0.9640200868536802,0.9839926204582717,0.9595663811844602,200,0.0030538325890119454,0.008476923657072453,0.016210899278690762,0.009102293092427282,200,3.900000000000002,5.0,7.0,5.245 +fresh-en-2,22,23,sqrtJS,16,full_attention,200,0,200,0.09078306090086699,0.23895533673930913,0.30426530304830524,0.22054257429139398,200,0.023469170182943346,0.055075300857424736,0.09086301401257516,0.058427924080751835,200,6.046355040516195,7.619268179768053,9.585119576904154,7.71124691949597,200,0.3778971900322622,0.4762042612355033,0.5990699735565096,0.48195293246849813,200,0.0446169532993613,0.0733306737639372,0.10978017114031512,0.0772446894596137,200,0.19773511533094978,0.2665851835263795,0.32795801759184384,0.26523541761548086,200,0.5384536096770526,0.6047399313588374,0.6823232703752686,0.6100649683213264,200,0.7721297210241375,0.8152846236318051,0.8642578252561324,0.8177328237062715,200,0.041858638850155665,0.06342044784640975,0.08085008476584839,0.062306743584434034,200,10.0,11.0,12.0,10.865 diff --git a/results/head-dominance/concentration-curve.csv b/results/head-dominance/concentration-curve.csv new file mode 100644 index 0000000..c276091 --- /dev/null +++ b/results/head-dominance/concentration-curve.csv @@ -0,0 +1,607 @@ +metric,fraction_ranked_heads,p10,median,p90 +Q,0.0,0.0,0.0,0.0 +Q,0.01,0.01737431848049164,0.024207810163497927,0.040043223857879645 +Q,0.02,0.03474863696098328,0.048415620326995853,0.08008644771575929 +Q,0.03,0.05212295544147492,0.07262343049049377,0.12012967157363894 +Q,0.04,0.06949727392196656,0.09683124065399171,0.16017289543151858 +Q,0.05,0.08687159240245819,0.12103905081748964,0.20021611928939823 +Q,0.06,0.10424591088294984,0.14524686098098755,0.24025934314727787 +Q,0.07,0.1211215353012085,0.16653286457061767,0.2672112132310868 +Q,0.08,0.13758879113197328,0.1866253399848938,0.2897918859720231 +Q,0.09,0.15409120869636533,0.2066990005970001,0.31256913959980015 +Q,0.1,0.1704423189163208,0.22684958279132844,0.33550719261169437 +Q,0.11,0.18685218268632892,0.24672719120979308,0.3587276176214219 +Q,0.12,0.20320073175430298,0.26654049336910246,0.3818341631889343 +Q,0.13,0.2185632281303406,0.2842993640899658,0.4009770392179489 +Q,0.14,0.23300261771678926,0.29968256413936617,0.4171214056015015 +Q,0.15,0.24738164186477662,0.31521878242492674,0.4339162802696229 +Q,0.16,0.2617198959589005,0.33065110981464385,0.4502217667102814 +Q,0.17,0.27588961946964263,0.3461452317237854,0.4671484246253968 +Q,0.18,0.28989240133762356,0.36176909863948825,0.48383628845214843 +Q,0.19,0.3038723428249359,0.37690478861331944,0.49967317867279054 +Q,0.2,0.31729141473770145,0.39086237251758577,0.5133090174198152 +Q,0.21,0.3308429169654846,0.40473467111587524,0.5271751768589019 +Q,0.22,0.34424016618728637,0.41847694337368013,0.5408917760848999 +Q,0.23,0.35749383318424227,0.43224998950958254,0.5541337549686433 +Q,0.24,0.3707662565708161,0.44598378241062164,0.5676164312362671 +Q,0.25,0.3840065896511078,0.4597482681274414,0.5813789308071137 +Q,0.26,0.3959834885597229,0.4717606556415558,0.59273619389534 +Q,0.27,0.40793633854389194,0.4838629364967346,0.6042708117961885 +Q,0.28,0.41990083503723147,0.4957883661985397,0.6155164992809296 +Q,0.29,0.43169007074832916,0.5077187478542328,0.6270478644371033 +Q,0.3,0.4436528295278549,0.5198427975177765,0.6385741269588471 +Q,0.31,0.45551044392585754,0.5318721652030944,0.6500959353446962 +Q,0.32,0.4669456263780594,0.54321497797966,0.6602133111953736 +Q,0.33,0.4783005748987198,0.5542978835105896,0.6699534819126131 +Q,0.34,0.489512093782425,0.5653712165355682,0.6796446554660798 +Q,0.35000000000000003,0.5007768940925599,0.5764826774597168,0.6897173345088959 +Q,0.36,0.5119028506278991,0.5876066887378693,0.6997994785308838 +Q,0.37,0.5230448249578475,0.5987306010723115,0.7098377628326416 +Q,0.38,0.5336348724365235,0.6092551350593567,0.7187074544429779 +Q,0.39,0.5439174950122833,0.6192927420139313,0.7270798141956332 +Q,0.4,0.5542610394954681,0.6291231215000153,0.7353755784034729 +Q,0.41000000000000003,0.5646437027454376,0.6390299117565155,0.7439257018566132 +Q,0.42,0.5748950352668762,0.6490671348571777,0.7523526072502136 +Q,0.43,0.5851743149757386,0.6589171504974365,0.7608411180973053 +Q,0.44,0.5953397953510284,0.6685371780395508,0.7689071040153503 +Q,0.45,0.6051786983013153,0.677568256855011,0.7761169922351837 +Q,0.46,0.6151465620994567,0.6867305719852448,0.7839144046306611 +Q,0.47000000000000003,0.6249501769542695,0.6958632254600525,0.7911958630084992 +Q,0.48,0.6349154961109161,0.7050456881523132,0.7986984915733337 +Q,0.49,0.6446966435909272,0.714237357378006,0.8061714801788331 +Q,0.5,0.6545684337615967,0.7233335077762604,0.813521409034729 +Q,0.51,0.6634617981910705,0.7313682210445405,0.8198267560005188 +Q,0.52,0.6725652790069581,0.7394421458244325,0.8261420361995697 +Q,0.53,0.6815666716098786,0.7475150763988495,0.832404525756836 +Q,0.54,0.6905837512016297,0.7556244969367982,0.8387663238048554 +Q,0.55,0.6994967448711396,0.7637638449668884,0.8448444211483003 +Q,0.56,0.7084609444141389,0.7719143807888031,0.851097755432129 +Q,0.5700000000000001,0.7170961196422577,0.7795686781406403,0.8568183658123018 +Q,0.58,0.7257474212646484,0.787055605649948,0.8623757457733154 +Q,0.59,0.7343528516292571,0.7944879209995269,0.8678745765686036 +Q,0.6,0.7428667426109313,0.8020348846912384,0.8736727249622345 +Q,0.61,0.751394653558731,0.8095412027835845,0.8792897477149964 +Q,0.62,0.7598428263664245,0.817003585100174,0.8849687988758088 +Q,0.63,0.7679615499973298,0.8239835953712463,0.8901845357418061 +Q,0.64,0.7756900873184204,0.8307234752178192,0.8947618057727814 +Q,0.65,0.783534129858017,0.837240344285965,0.8993717551231385 +Q,0.66,0.79137437748909,0.843800311088562,0.9040085859298707 +Q,0.67,0.7991621415615082,0.8503664135932922,0.9089032695293426 +Q,0.68,0.8068775184154511,0.8569483280181884,0.9138754611015321 +Q,0.6900000000000001,0.81456493973732,0.8633548069000244,0.9184835515022278 +Q,0.7000000000000001,0.8220234143733979,0.8694061756134034,0.9226034843921662 +Q,0.71,0.8295537467002868,0.8754319489002227,0.9266780784130096 +Q,0.72,0.8370579061508179,0.8815048468112945,0.930839075088501 +Q,0.73,0.8444442536830902,0.8876474845409393,0.9349904763698579 +Q,0.74,0.8517861280441283,0.8937022399902343,0.9392964746952057 +Q,0.75,0.8590913534164428,0.8997131586074829,0.9435038268566132 +Q,0.76,0.8657438101768494,0.9048986363410949,0.9468678944110871 +Q,0.77,0.8724173781871796,0.910088735818863,0.9502578182220459 +Q,0.78,0.8789712898731232,0.9152951538562775,0.953789341211319 +Q,0.79,0.8854904797077179,0.9204946565628052,0.9572178978919983 +Q,0.8,0.891982980966568,0.9256745159626008,0.9608542692661286 +Q,0.81,0.8984787909984588,0.9308709681034089,0.9645308711528778 +Q,0.8200000000000001,0.904705398797989,0.9356072580814362,0.9675366463661194 +Q,0.8300000000000001,0.9108241984844209,0.9402387380599976,0.9703989825248719 +Q,0.84,0.9170105950832367,0.9447744297981262,0.9733367812633514 +Q,0.85,0.9230920946598052,0.9493771553039552,0.976245310306549 +Q,0.86,0.9291698956489562,0.9540431928634643,0.9791290521621705 +Q,0.87,0.9352708640098572,0.9585881996154785,0.9820211317539215 +Q,0.88,0.9408319525718689,0.9626672923564912,0.9843749580383301 +Q,0.89,0.9459624199867249,0.9662620210647583,0.9860894491672516 +Q,0.9,0.9510724818706513,0.96986643075943,0.9878358781337738 +Q,0.91,0.9562040007114411,0.9734911298751832,0.9896683590412141 +Q,0.92,0.9613230566978455,0.9771507596969604,0.9915475554466248 +Q,0.93,0.9663616073131561,0.9807081019878388,0.9934325048923492 +Q,0.9400000000000001,0.9713375053405762,0.9840153193473815,0.9950843453407288 +Q,0.9500000000000001,0.9761145877838135,0.9866794228553772,0.9959036231040955 +Q,0.96,0.9808916702270508,0.9893435263633727,0.9967229008674622 +Q,0.97,0.9856687526702881,0.9920076298713685,0.9975421786308288 +Q,0.98,0.9904458351135254,0.994671733379364,0.9983614563941956 +Q,0.99,0.9952229175567627,0.9973358368873596,0.999180781364441 +Q,1.0,0.9999998807907104,1.0,1.0000001192092896 +K,0.0,0.0,0.0,0.0 +K,0.01,0.01654582941532135,0.023448383808135985,0.039481280565261845 +K,0.02,0.0330916588306427,0.04689676761627197,0.07896256113052369 +K,0.03,0.04963748824596405,0.07034515142440795,0.11844384169578552 +K,0.04,0.0661833176612854,0.09379353523254394,0.15792512226104738 +K,0.05,0.08272914707660675,0.11724191904067993,0.19740640282630922 +K,0.06,0.0992749764919281,0.1406903028488159,0.23688768339157104 +K,0.07,0.1154492797255516,0.16131282150745394,0.2636175501346588 +K,0.08,0.13137356448173523,0.1809262776374817,0.28626059031486517 +K,0.09,0.14729287779331207,0.2005068141222,0.30902923345565797 +K,0.1,0.1631060603260994,0.21992731988430025,0.3321889847517014 +K,0.11,0.17888272017240525,0.2393816989660263,0.35505529940128333 +K,0.12,0.19464654493331907,0.25845629155635835,0.378376546382904 +K,0.13,0.20965741169452667,0.27578357636928563,0.3976377201080322 +K,0.14,0.22374130892753605,0.2914238291978836,0.41461005926132205 +K,0.15,0.23782149672508238,0.3068800508975983,0.43140474140644075 +K,0.16,0.25175367575883867,0.3223154044151306,0.44771281838417054 +K,0.17,0.26563248360157016,0.3378386235237122,0.4644449923038483 +K,0.18,0.2796056665182114,0.3530931955575943,0.4817081689834596 +K,0.19,0.2932958579063416,0.3681097024679184,0.49799002408981324 +K,0.2,0.3067180436849594,0.38210111260414126,0.5115388381481173 +K,0.21,0.3199918949604035,0.39623517155647275,0.5249912676811218 +K,0.22,0.3333051850795746,0.4102067911624908,0.5387681150436401 +K,0.23,0.34653794288635253,0.42409759104251865,0.5528232893943786 +K,0.24,0.35974573945999144,0.4380839931964874,0.5664957237243653 +K,0.25,0.37291785776615144,0.4519532322883606,0.5801074922084809 +K,0.26,0.384872496008873,0.4642051994800568,0.5911368012428284 +K,0.27,0.39681819951534275,0.47636784374713903,0.6021544194221496 +K,0.28,0.40870070672035225,0.4885217934846878,0.6135927557945252 +K,0.29,0.4205945941209793,0.5005989354848861,0.6241933698654176 +K,0.3,0.4325110739469528,0.5127486765384675,0.6355258464813233 +K,0.31,0.4442179522514343,0.5247822642326355,0.6468103218078614 +K,0.32,0.45583432936668394,0.5362907946109772,0.6569152936935425 +K,0.33,0.4673587625026703,0.5474274563789367,0.6670719122886658 +K,0.34,0.47873131132125857,0.5587722098827362,0.6774317479133606 +K,0.35000000000000003,0.49004184484481816,0.5698711425065994,0.6873960781097412 +K,0.36,0.5014507284164428,0.5810222160816192,0.6975013160705567 +K,0.37,0.5127376890182496,0.5921002018451691,0.7073392987251281 +K,0.38,0.5235232629776001,0.6026276862621307,0.7162758450508119 +K,0.39,0.5339364340305328,0.6124646723270416,0.7249888377189637 +K,0.4,0.5444318759441377,0.6223917484283448,0.7333481669425964 +K,0.41000000000000003,0.5548466427326203,0.6322715127468109,0.7420074286460877 +K,0.42,0.5651789243221282,0.642231079339981,0.7505033988952636 +K,0.43,0.5754836821556091,0.6520547556877136,0.7586206340789795 +K,0.44,0.5857347345352173,0.6619505846500396,0.7667404103279115 +K,0.45,0.595584272146225,0.6711941003799439,0.7740319669246674 +K,0.46,0.6055839660167694,0.6804516804218292,0.7814083921909333 +K,0.47000000000000003,0.6155420253276825,0.6897563314437867,0.788986647129059 +K,0.48,0.6254948527812957,0.6990441572666168,0.7967307019233704 +K,0.49,0.6354581391811371,0.7082551157474517,0.8044977717399598 +K,0.5,0.6453050136566162,0.7175862789154053,0.8124186098575592 +K,0.51,0.6544364266395569,0.7258522117137909,0.8187114622592927 +K,0.52,0.6634779360294343,0.7339593029022217,0.8250258047580719 +K,0.53,0.6724951410293579,0.7421017563343049,0.8311313891410828 +K,0.54,0.6815666937828064,0.7503498685359955,0.8374199008941651 +K,0.55,0.6906708002090455,0.7584902405738831,0.8436271393299104 +K,0.56,0.6997027945518494,0.7666817247867586,0.8499409723281861 +K,0.5700000000000001,0.7084085998535157,0.7744602155685425,0.855674866437912 +K,0.58,0.7171198775768279,0.782093459367752,0.8611813712120056 +K,0.59,0.7259483613967895,0.7897086620330811,0.8667407891750335 +K,0.6,0.7347933828830718,0.7973713934421539,0.872250589132309 +K,0.61,0.7434831933975219,0.804972927570343,0.878096629858017 +K,0.62,0.7521167180538177,0.8126032769680023,0.8837950630187988 +K,0.63,0.7605078415870666,0.8197759139537811,0.888842754840851 +K,0.64,0.7684811193943024,0.8264838278293609,0.8935688529014587 +K,0.65,0.7765250539779663,0.8332662999629974,0.8981407761573792 +K,0.66,0.7844998223781586,0.839933078289032,0.9029493284225464 +K,0.67,0.7923307213783264,0.8466307187080384,0.9077160177230835 +K,0.68,0.8002330765724183,0.8533342480659485,0.9124865460395813 +K,0.6900000000000001,0.8081206042766572,0.8598504793643952,0.9172539837360382 +K,0.7000000000000001,0.8157313001155854,0.8660956025123596,0.9214256882667542 +K,0.71,0.8233990607261658,0.8722559475898742,0.9255271220207214 +K,0.72,0.8309884355068207,0.8783128225803375,0.9297294902801514 +K,0.73,0.8385065405368805,0.8844152855873108,0.9339916477203369 +K,0.74,0.846130084991455,0.8906441295146943,0.9383736855983734 +K,0.75,0.8537726521492004,0.8967804312705994,0.9426405429840088 +K,0.76,0.8605813899040222,0.9020169365406037,0.9458947782516479 +K,0.77,0.8673497393131256,0.9073439812660218,0.9494328136444092 +K,0.78,0.8739968962669373,0.9126505196094513,0.9529471206665039 +K,0.79,0.8807190537452698,0.9179106068611145,0.9565000057220459 +K,0.8,0.8873664689064027,0.9231710612773896,0.9600775682926178 +K,0.81,0.893988229751587,0.9284985029697419,0.9638077754974366 +K,0.8200000000000001,0.900476526260376,0.9334009754657746,0.9668706696033479 +K,0.8300000000000001,0.9068691065311432,0.9381741273403168,0.9697751314640045 +K,0.84,0.9132789044380188,0.9429380714893341,0.9727330930233001 +K,0.85,0.9196463751792907,0.9476843893527984,0.9755963099002839 +K,0.86,0.925910506248474,0.9524028086662293,0.9784519300460816 +K,0.87,0.9321331479549407,0.9572086131572723,0.981397498369217 +K,0.88,0.9378810062408447,0.9614343118667603,0.9837687277793884 +K,0.89,0.9432600319385529,0.9651312446594238,0.9855939209461212 +K,0.9,0.9486236989498138,0.9688000857830048,0.9873944270610809 +K,0.91,0.9540016658306122,0.9725240027904511,0.9891698353290558 +K,0.92,0.9593397746086121,0.9762697958946227,0.9911036093235016 +K,0.93,0.9646305012702943,0.9800131964683533,0.9930221819877625 +K,0.9400000000000001,0.9698303661346436,0.9834258210659027,0.9947616651058198 +K,0.9500000000000001,0.9748586225509643,0.986188179254532,0.9956347119808198 +K,0.96,0.9798868789672852,0.9889505374431611,0.9965077588558198 +K,0.97,0.984915135383606,0.9917128956317902,0.9973808057308198 +K,0.98,0.9899434452056884,0.9944752538204193,0.9982538526058197 +K,0.99,0.9949717187881469,0.9972376155853271,0.9991268994808197 +K,1.0,0.9999998807907104,1.0,1.0000001192092896 +V,0.0,0.0,0.0,0.0 +V,0.01,0.017283772349357605,0.02430927038192749,0.04318253946304324 +V,0.02,0.03456754469871521,0.04861854076385498,0.08636507892608648 +V,0.03,0.05185131704807281,0.07292781114578246,0.1295476183891297 +V,0.04,0.06913508939743042,0.09723708152770996,0.17273015785217297 +V,0.05,0.08641886174678803,0.12154635190963746,0.21591269731521617 +V,0.06,0.10370263409614562,0.14585562229156493,0.2590952367782594 +V,0.07,0.1204776572585106,0.1674258816242218,0.2863244551420212 +V,0.08,0.1368465223312378,0.18790233314037325,0.30851807403564463 +V,0.09,0.15343041354417802,0.20825985312461853,0.33073699593544004 +V,0.1,0.1699641725420952,0.22841397523880005,0.3534713393449784 +V,0.11,0.18619864618778229,0.24811343789100648,0.3777024531364441 +V,0.12,0.20233429896831512,0.2680304443836212,0.40027050733566283 +V,0.13,0.21771681332588197,0.2860366332530975,0.41997874963283544 +V,0.14,0.2321199409365654,0.30205620050430304,0.4362629147768022 +V,0.15,0.24663646221160887,0.3181026101112366,0.452704194188118 +V,0.16,0.26091460835933683,0.3340769386291504,0.4687001596689225 +V,0.17,0.2751953773498535,0.34998053133487705,0.4849494593143466 +V,0.18,0.289476178586483,0.36573639690876003,0.5020644532442093 +V,0.19,0.3034248900413513,0.38115279197692875,0.5177486491203308 +V,0.2,0.31686339497566224,0.3954771369695663,0.5314432525634766 +V,0.21,0.3303861356973648,0.409612797498703,0.5450934979915619 +V,0.22,0.34372555625438694,0.42380967140197756,0.558682193517685 +V,0.23,0.3571029105186463,0.43807668566703795,0.5721341028213501 +V,0.24,0.37046848487854,0.4522347277402878,0.5861496005058289 +V,0.25,0.38361903131008146,0.46644602715969086,0.599850219488144 +V,0.26,0.3955929044485092,0.4787029314041138,0.6115220782756805 +V,0.27,0.4073226959705353,0.49086178421974186,0.6222544310092927 +V,0.28,0.41961391782760626,0.5030666255950929,0.6332944786548615 +V,0.29,0.431655185341835,0.5154197633266449,0.644315850019455 +V,0.3,0.4436744874715805,0.5276248574256897,0.654893572330475 +V,0.31,0.45549591922760013,0.5398323786258697,0.6656730680465699 +V,0.32,0.46702083706855774,0.5512767922878266,0.675122779607773 +V,0.33,0.47842310690879825,0.5625461876392365,0.6847947258949281 +V,0.34,0.4897958809137345,0.5739054489135742,0.6946210875511171 +V,0.35000000000000003,0.5010297298431398,0.5849657893180849,0.7043648838996888 +V,0.36,0.5123609721660614,0.5962558937072754,0.7139363582134247 +V,0.37,0.5237813906669617,0.6073624992370605,0.7238476750850678 +V,0.38,0.5345841057300568,0.6177962017059326,0.7329005780220033 +V,0.39,0.5449665372371674,0.6276165914535523,0.740968234539032 +V,0.4,0.555171777009964,0.6376343905925751,0.7490898060798645 +V,0.41000000000000003,0.5653766236305237,0.6475298070907594,0.7572771577835083 +V,0.42,0.575601366519928,0.6573278522491455,0.7654482088088989 +V,0.43,0.585959151506424,0.6672149848937988,0.7737478222846985 +V,0.44,0.5961243166923523,0.6768707811832428,0.7815244855880737 +V,0.45,0.6060832393169403,0.6859987318515778,0.7885640394687653 +V,0.46,0.6159256119728088,0.6950290143489838,0.7957454979419709 +V,0.47000000000000003,0.6256353540420533,0.7039893877506256,0.8030814847946168 +V,0.48,0.6356671802997589,0.7131526625156402,0.8104326679706575 +V,0.49,0.645386251449585,0.7223494780063628,0.817746812582016 +V,0.5,0.6551911294460296,0.7314304113388062,0.8252103924751282 +V,0.51,0.6642950916290283,0.7392143762111665,0.8312481815814972 +V,0.52,0.6732384381294251,0.7471807074546815,0.8374078998565675 +V,0.53,0.6823097643852234,0.7552064681053162,0.8434679939746857 +V,0.54,0.6912216863632202,0.7631723773479462,0.8492499709129334 +V,0.55,0.7000860726833343,0.7712316632270814,0.8549364078044892 +V,0.56,0.7091048576831819,0.7791959095001222,0.8608052072525025 +V,0.5700000000000001,0.7178576200008393,0.7867359852790833,0.8663183281421662 +V,0.58,0.7265823242664337,0.7941215205192566,0.8716831486225128 +V,0.59,0.7351070475578307,0.8015416729450225,0.8766904950141906 +V,0.6,0.7437374496459961,0.8089299738407135,0.8820634484291078 +V,0.61,0.7523676345348358,0.8162679362297058,0.8877835907936097 +V,0.62,0.7609670441150664,0.8237060403823853,0.8934823803901674 +V,0.63,0.7692259292602539,0.8305854356288911,0.8981908988952637 +V,0.64,0.7769662261009217,0.8370555770397186,0.9026195304393769 +V,0.65,0.7846354603767395,0.8434919118881226,0.9070481300354004 +V,0.66,0.7925256202220916,0.8498201990127563,0.9115287780761719 +V,0.67,0.8002883973121644,0.8563213026523591,0.9161587166786194 +V,0.68,0.8080831336975098,0.8626770615577698,0.9207553291320801 +V,0.6900000000000001,0.8156747574806213,0.8689885532855988,0.9251275277137757 +V,0.7000000000000001,0.8231072735786439,0.8749236524105073,0.9289538967609406 +V,0.71,0.8305649266242981,0.8808118867874145,0.9330252027511596 +V,0.72,0.838040625333786,0.8867127418518066,0.9369128992557526 +V,0.73,0.8453985905647278,0.892595225572586,0.9409217324256898 +V,0.74,0.8527897372245788,0.8985521054267883,0.9449429907798768 +V,0.75,0.8601445257663727,0.9045099020004272,0.9489896416664123 +V,0.76,0.8667308814525604,0.9095117509365082,0.9521236038208007 +V,0.77,0.8731506879329681,0.9145371973514558,0.9552511143684387 +V,0.78,0.879665254354477,0.9195238816738129,0.9583617885112762 +V,0.79,0.8862169976234436,0.9245164513587952,0.9616534554958344 +V,0.8,0.8926905369758605,0.9295362174510956,0.9648536276817322 +V,0.81,0.8991780304908753,0.9344788241386414,0.9681034808158875 +V,0.8200000000000001,0.9054281823635102,0.938979573249817,0.9708296427726746 +V,0.8300000000000001,0.9115218710899353,0.9433979380130768,0.9734626023769378 +V,0.84,0.9176776471138001,0.947852623462677,0.9759971308708191 +V,0.85,0.9236609935760498,0.9523427069187165,0.9786101937294006 +V,0.86,0.929715832710266,0.9567157697677613,0.9812671732902527 +V,0.87,0.9357648379802704,0.9611148822307587,0.9839867370128632 +V,0.88,0.9412867667675018,0.9649768519401549,0.9860567011833191 +V,0.89,0.9464687509536743,0.9683588182926177,0.9875812561511994 +V,0.9,0.9515495038032532,0.971798586845398,0.9891605794429779 +V,0.91,0.9565999655723572,0.9751971471309662,0.9907485103607178 +V,0.92,0.961646210193634,0.9785968708992004,0.9924309463500978 +V,0.93,0.9667050132751466,0.9819891083240508,0.99413436460495 +V,0.9400000000000001,0.9716645660400391,0.9850796353816986,0.9956805167198182 +V,0.9500000000000001,0.9763871383666992,0.9875663578510284,0.996400420665741 +V,0.96,0.9811097106933594,0.9900530803203582,0.9971203246116638 +V,0.97,0.9858322830200195,0.9925398027896881,0.9978402285575867 +V,0.98,0.9905548553466798,0.995026525259018,0.9985601325035095 +V,0.99,0.9952774276733398,0.9975132477283477,0.9992800364494324 +V,1.0,0.9999998807907104,1.0,1.0000001192092896 +QKV,0.0,0.0,0.0,0.0 +QKV,0.01,0.016800588846206665,0.022913488149642947,0.039115326166152965 +QKV,0.02,0.03360117769241333,0.04582697629928589,0.07823065233230593 +QKV,0.03,0.05040176653861999,0.06874046444892884,0.11734597849845889 +QKV,0.04,0.06720235538482666,0.09165395259857179,0.15646130466461186 +QKV,0.05,0.08400294423103333,0.11456744074821473,0.19557663083076485 +QKV,0.06,0.10080353307723998,0.13748092889785768,0.23469195699691778 +QKV,0.07,0.11710977983474732,0.15784060776233674,0.25975488471984864 +QKV,0.08,0.1332212793827057,0.17726378619670868,0.2813613904714585 +QKV,0.09,0.1492583650946617,0.19674843072891235,0.3026642986536028 +QKV,0.1,0.1650717580318451,0.21603886485099794,0.3245760136842728 +QKV,0.11,0.1810185403227806,0.23529659569263459,0.346066223025322 +QKV,0.12,0.1968268607854843,0.25457505643367767,0.3676545374393464 +QKV,0.13,0.21185547560453416,0.27170712769031524,0.38651670348644257 +QKV,0.14,0.22593659472465516,0.28702558994293215,0.4029968308210373 +QKV,0.15,0.24001431673765183,0.30246447026729584,0.4196414488554001 +QKV,0.16,0.25393714344501495,0.3177560770511627,0.43614139401912694 +QKV,0.17,0.2679267185926437,0.3329687884449959,0.45319731128215796 +QKV,0.18,0.2817780703306198,0.3484531509876251,0.4696514335870744 +QKV,0.19,0.29557060432434085,0.3632785528898239,0.4859631145000458 +QKV,0.2,0.3088128769397736,0.3772181421518326,0.499290314912796 +QKV,0.21,0.3220159499645233,0.3911362951993942,0.5133361501693726 +QKV,0.22,0.335122153878212,0.40502499282360077,0.5274076409339904 +QKV,0.23,0.3482807829380035,0.41881872832775113,0.5412303044795991 +QKV,0.24,0.3612290899753571,0.4325333762168884,0.5552990515232086 +QKV,0.25,0.37423557639122007,0.4464688301086426,0.5689850687980652 +QKV,0.26,0.38623303866386416,0.45865653634071346,0.5801734836101533 +QKV,0.27,0.39808811724185944,0.4707050800323487,0.5909626884460449 +QKV,0.28,0.41001566576957704,0.48278572440147405,0.6020210313796998 +QKV,0.29,0.42166742420196535,0.49481430590152736,0.6131852295398712 +QKV,0.3,0.43341935276985166,0.5068297147750854,0.6243260729312897 +QKV,0.31,0.44513567411899563,0.5190878522396087,0.6355959362983703 +QKV,0.32,0.45652735149860385,0.5305153799057007,0.6455700662136078 +QKV,0.33,0.4678807412385941,0.5416884338855743,0.6556204934120178 +QKV,0.34,0.4793121231794357,0.5528202247619629,0.665543577194214 +QKV,0.35000000000000003,0.49052130520343784,0.5641218602657319,0.6755343472957612 +QKV,0.36,0.5018073704242706,0.5752099800109863,0.6852814710140229 +QKV,0.37,0.5130722978115082,0.5863845479488373,0.6953443140983582 +QKV,0.38,0.523883184671402,0.5969428026676178,0.704551686525345 +QKV,0.39,0.5342232055664063,0.6070881199836731,0.7130267243385315 +QKV,0.4,0.5446159029006958,0.6171286404132843,0.7214405262470246 +QKV,0.41000000000000003,0.5548589320182801,0.6271740937232971,0.7302109038829804 +QKV,0.42,0.5652168192863465,0.6371716094017028,0.7388426251411438 +QKV,0.43,0.5754696817398071,0.6471431612968446,0.7473425850868226 +QKV,0.44,0.5857338607311249,0.6570617055892944,0.7557081656455994 +QKV,0.45,0.59575688123703,0.6663605511188506,0.763181256055832 +QKV,0.46,0.6056944172382356,0.6757081687450409,0.7707117874622345 +QKV,0.47000000000000003,0.6155292620658875,0.6849586820602418,0.778394143342972 +QKV,0.48,0.625439323425293,0.6942036569118499,0.7862356214523315 +QKV,0.49,0.6352029254436492,0.7035364890098572,0.7940804347991943 +QKV,0.5,0.6450497508049011,0.712812602519989,0.8019364178180695 +QKV,0.51,0.6540212242603302,0.7210712790489197,0.8082412190437317 +QKV,0.52,0.663158641576767,0.7293519377708435,0.8146322522163392 +QKV,0.53,0.6721282997131348,0.7375253987312317,0.8210127611160278 +QKV,0.54,0.6811432604789734,0.7457767009735108,0.8274090337753296 +QKV,0.55,0.6901337695121765,0.7540464878082276,0.833967343568802 +QKV,0.56,0.6991206421852113,0.7623445355892182,0.8404466843605043 +QKV,0.5700000000000001,0.7080617368221284,0.7701746380329133,0.8463437986373902 +QKV,0.58,0.7167876584529876,0.7778467082977294,0.8521203808784485 +QKV,0.59,0.7256710972785949,0.7855744612216948,0.858158596277237 +QKV,0.6,0.7344047844409942,0.7932448446750641,0.8639399921894073 +QKV,0.61,0.7430357675552368,0.8009650802612305,0.8697287745475769 +QKV,0.62,0.7517478482723237,0.8086153268814087,0.8755806272029877 +QKV,0.63,0.7601338062286377,0.815826096534729,0.8809755599498749 +QKV,0.64,0.7681746380329133,0.8225968682765961,0.885839326620102 +QKV,0.65,0.7760845637321472,0.8293470144271851,0.8907229280471802 +QKV,0.66,0.7840023334026337,0.8361264383792877,0.8956864068508148 +QKV,0.67,0.79203378033638,0.8428762078285217,0.9007084641456604 +QKV,0.68,0.7998658556938172,0.8496367418766022,0.9059668192863464 +QKV,0.6900000000000001,0.8076452324390412,0.8562417566776276,0.9107926976680756 +QKV,0.7000000000000001,0.8153099548816681,0.8626011312007904,0.9153185081481934 +QKV,0.71,0.8229050102233887,0.8689090728759765,0.9197219483852386 +QKV,0.72,0.830444237947464,0.8752207767963409,0.9241207015514373 +QKV,0.73,0.83803036570549,0.8814849865436554,0.9287123126983643 +QKV,0.74,0.8456049251556397,0.8876647126674653,0.933222519159317 +QKV,0.75,0.8531072556972503,0.8940083384513855,0.9378141283988953 +QKV,0.76,0.8599213063716888,0.89940305352211,0.9415755398273469 +QKV,0.77,0.8667151927947998,0.9048697185516357,0.9452722387313843 +QKV,0.78,0.8734789602756501,0.9102456116676331,0.9489993734359742 +QKV,0.79,0.8802480485439301,0.9157090425491333,0.9528656978607178 +QKV,0.8,0.8869937431812287,0.9211421310901642,0.9568616950511932 +QKV,0.81,0.8936453433036805,0.926614590883255,0.9607337646484375 +QKV,0.8200000000000001,0.9000910522937775,0.9315775275230408,0.9640683367252351 +QKV,0.8300000000000001,0.906400206565857,0.936442666053772,0.9671617360115052 +QKV,0.84,0.9127689161300659,0.9412961721420288,0.970330352306366 +QKV,0.85,0.9191430509090424,0.9461468100547791,0.9735007870197296 +QKV,0.86,0.9254219424724579,0.9510287928581238,0.9766703248023987 +QKV,0.87,0.931663346529007,0.955849256515503,0.9798700177669526 +QKV,0.88,0.9374851126670838,0.960194821357727,0.9824348247051239 +QKV,0.89,0.9429919452667237,0.9640263199806214,0.9844272830486298 +QKV,0.9,0.948404278755188,0.9679100811481476,0.9864511251449585 +QKV,0.91,0.953758846282959,0.9717504739761353,0.9884686648845673 +QKV,0.92,0.9591385016441345,0.9755612909793854,0.9904830720424652 +QKV,0.93,0.9644818317890167,0.9792894005775452,0.9926027362346649 +QKV,0.9400000000000001,0.9697480845451355,0.9827727532386781,0.9944333798885346 +QKV,0.9500000000000001,0.974790072441101,0.9856439709663392,0.9953611409664155 +QKV,0.96,0.9798320603370666,0.9885151886940002,0.9962889020442963 +QKV,0.97,0.9848740482330323,0.9913864064216613,0.9972166631221772 +QKV,0.98,0.9899160361289978,0.9942576241493225,0.9981444263458251 +QKV,0.99,0.9949580240249634,0.997128803730011,0.9990722131729126 +QKV,1.0,0.9999998807907104,1.0,1.0000001192092896 +JS,0.0,0.0,0.0,0.0 +JS,0.01,0.03884871459007263,0.06768969774246217,0.11432822799682622 +JS,0.02,0.07769742918014526,0.13537939548492434,0.22865645599365245 +JS,0.03,0.11654614377021788,0.20306909322738648,0.34298468399047866 +JS,0.04,0.15539485836029052,0.2707587909698487,0.4573129119873049 +JS,0.05,0.19424357295036318,0.3384484887123108,0.571641139984131 +JS,0.06,0.23309228754043576,0.40613818645477295,0.6859693679809573 +JS,0.07,0.2657834422588348,0.45004083395004274,0.7273755989074708 +JS,0.08,0.2954695240259171,0.4842478346824646,0.7477391128540042 +JS,0.09,0.3247961348295212,0.5185334014892578,0.768332978963852 +JS,0.1,0.3545645266771316,0.5515236675739288,0.7914890015125274 +JS,0.11,0.38462327623367315,0.58460333943367,0.8137568414211275 +JS,0.12,0.41278298902511595,0.6181776452064515,0.8365184249877929 +JS,0.13,0.4381288493871689,0.6445403957366944,0.8526345763206483 +JS,0.14,0.4601372056007385,0.6621946942806245,0.8608411037921907 +JS,0.15,0.4830065017938614,0.6811392307281494,0.8691276741027832 +JS,0.16,0.5059864231348038,0.6999038302898407,0.8784728331565859 +JS,0.17,0.5269229650497438,0.7178922951221467,0.8874182481765748 +JS,0.18,0.5491097736358642,0.736287124156952,0.8962645514011384 +JS,0.19,0.569173704624176,0.7535254609584808,0.9042517039775848 +JS,0.2,0.5854026472568513,0.7646608114242555,0.9088869571685791 +JS,0.21,0.6002584474086761,0.7762962257862092,0.9133361880779266 +JS,0.22,0.6166204824447632,0.7885232996940612,0.9179978444576264 +JS,0.23,0.6338017940521241,0.8002816021442414,0.9229386105537416 +JS,0.24,0.6494297926425934,0.8115583264827728,0.9274915595054627 +JS,0.25,0.6646065950393677,0.8229818642139435,0.9326551020145416 +JS,0.26,0.6766561505794525,0.830416167974472,0.9354733812808991 +JS,0.27,0.6890899057388307,0.8385554373264312,0.9384642949104309 +JS,0.28,0.7011624486446382,0.8459331226348877,0.9414866840839388 +JS,0.29,0.7136204130649567,0.8537040495872498,0.9447876868247986 +JS,0.3,0.7261769020557404,0.8608507156372069,0.948199735879898 +JS,0.31,0.737364771604538,0.8685437583923339,0.9513335609436036 +JS,0.32,0.7474184038639069,0.8747290325164795,0.9536683037281036 +JS,0.33,0.7567795021533966,0.8799037206172944,0.9556315634250641 +JS,0.34,0.7661785423755646,0.8851364481449127,0.9576984004974366 +JS,0.35000000000000003,0.7759240758419037,0.8906245470046997,0.9600424611568451 +JS,0.36,0.7850462753772736,0.8958927774429322,0.9621104860305787 +JS,0.37,0.7944081070423126,0.9015586459636689,0.9642150843143463 +JS,0.38,0.8023757503032685,0.9061590147018432,0.9660220458507538 +JS,0.39,0.8092640779018402,0.9101581311225891,0.9676647691726684 +JS,0.4,0.8173421382904053,0.9139463424682617,0.9692228877544403 +JS,0.41000000000000003,0.8247527976036072,0.9178964149951935,0.9707063846588134 +JS,0.42,0.8322649049758911,0.9219050741195679,0.9721290996074677 +JS,0.43,0.8392793273925782,0.9257989919185639,0.9737422184944152 +JS,0.44,0.8456548821926116,0.92972390294075,0.9752977676391601 +JS,0.45,0.8516635155677796,0.9326756596565247,0.976392627954483 +JS,0.46,0.8576600644588471,0.9356075501441956,0.9774714715480805 +JS,0.47000000000000003,0.8636316668987274,0.9386734962463379,0.9786568965911866 +JS,0.48,0.8694883847236633,0.9416249728202819,0.9797335259914398 +JS,0.49,0.8749428472518921,0.9446432602405548,0.980886360168457 +JS,0.5,0.8803883373737336,0.9476926624774933,0.9820005357265472 +JS,0.51,0.8849196114540101,0.9500687575340271,0.9828275144100189 +JS,0.52,0.8898923082351685,0.9524643468856812,0.9836523883342743 +JS,0.53,0.8945713617801666,0.9547911870479584,0.9844611814022064 +JS,0.54,0.8990499572753906,0.9570967531204224,0.9852430467605592 +JS,0.55,0.9031755709648133,0.9594007134437561,0.9861137783527374 +JS,0.56,0.9077128639221191,0.9617905187606812,0.9869850125312805 +JS,0.5700000000000001,0.9117892420291901,0.9636707937717438,0.9877335991859436 +JS,0.58,0.9158590860366821,0.9654559636116028,0.9883272135257721 +JS,0.59,0.9196345641613006,0.9673038864135742,0.9888953006267547 +JS,0.6,0.9233547019958497,0.9690265655517578,0.9895506215095521 +JS,0.61,0.9272808990478516,0.9708032882213593,0.9902336106300353 +JS,0.62,0.930694507598877,0.9726557207107545,0.9908595650196076 +JS,0.63,0.9339826872348785,0.974253672361374,0.9914292964935303 +JS,0.64,0.9372651960849762,0.9756282830238343,0.9918948097229005 +JS,0.65,0.9402425110340119,0.9770334780216217,0.9923793697357177 +JS,0.66,0.9436105864048004,0.9784448826313019,0.9929312632083893 +JS,0.67,0.9466121823787689,0.9798834145069122,0.993453537940979 +JS,0.68,0.9498065371513368,0.9812905657291413,0.993977930545807 +JS,0.6900000000000001,0.9528964278697968,0.9826846134662628,0.9944403920173644 +JS,0.7000000000000001,0.9554721260070801,0.9837263226509094,0.9947766947746276 +JS,0.71,0.9579216969013213,0.9847511875629424,0.9951292600631714 +JS,0.72,0.9605836656093597,0.9857768499851227,0.9954641854763031 +JS,0.73,0.963028039932251,0.9867740631103517,0.9957879407405853 +JS,0.74,0.9654854700565338,0.987831848859787,0.9961545026302338 +JS,0.75,0.9679702877998352,0.9889267385005951,0.9965402781963348 +JS,0.76,0.9699831779003143,0.9897364008426666,0.9967825655937195 +JS,0.77,0.9718249397277832,0.9904715871810914,0.997047091960907 +JS,0.78,0.9737954559326172,0.9912588238716125,0.9973083426952363 +JS,0.79,0.9758035860061646,0.9920175731182099,0.9975660495758056 +JS,0.8,0.9776474750041961,0.9927996754646301,0.9978384554386139 +JS,0.81,0.9795894579887391,0.9935428524017333,0.9981349158287048 +JS,0.8200000000000001,0.981195797920227,0.9941351675987243,0.9983249680995941 +JS,0.8300000000000001,0.9827687723636628,0.9946750748157501,0.9984824681282044 +JS,0.84,0.9842416610717774,0.9952229368686676,0.9986519126892089 +JS,0.85,0.9858171153068542,0.9957622587680817,0.9988238537311555 +JS,0.86,0.9873283450603485,0.9963006126880645,0.9990038769245148 +JS,0.87,0.9887004697322846,0.996846137046814,0.9991918690204621 +JS,0.88,0.9899477322101593,0.9972608757019044,0.9993443593978881 +JS,0.89,0.9909535527229308,0.9975963270664215,0.9994242010116577 +JS,0.9,0.9920106387138367,0.9979216873645782,0.9995087313652038 +JS,0.91,0.993091065645218,0.998242930173874,0.9995891172885895 +JS,0.92,0.9942101104259492,0.9985765779018403,0.9996774969100952 +JS,0.93,0.9952523741722107,0.9989218008518219,0.9997698504924775 +JS,0.9400000000000001,0.9961830954551697,0.9992191421985626,0.999866527557373 +JS,0.9500000000000001,0.996819236278534,0.9993492901325225,0.9998887729644775 +JS,0.96,0.9974553771018982,0.9994794380664825,0.999911018371582 +JS,0.97,0.9980915179252624,0.9996095860004426,0.9999332637786865 +JS,0.98,0.9987276587486267,0.9997397315502167,0.999955509185791 +JS,0.99,0.999363799571991,0.9998698616027832,0.9999777846336364 +JS,1.0,0.9999998807907104,1.0,1.0000001192092896 +sqrtJS,0.0,0.0,0.0,0.0 +sqrtJS,0.01,0.023318491220474243,0.03479416608810425,0.05526199054718017 +sqrtJS,0.02,0.046636982440948486,0.0695883321762085,0.11052398109436035 +sqrtJS,0.03,0.06995547366142273,0.10438249826431274,0.16578597164154052 +sqrtJS,0.04,0.09327396488189697,0.139176664352417,0.2210479621887207 +sqrtJS,0.05,0.11659245610237123,0.17397083044052125,0.27630995273590087 +sqrtJS,0.06,0.13991094732284545,0.20876499652862548,0.33157194328308104 +sqrtJS,0.07,0.16095035839080812,0.2352624160051346,0.36345391547679917 +sqrtJS,0.08,0.18119837176799775,0.26024092972278595,0.38791794967651366 +sqrtJS,0.09,0.20109880912303923,0.28399891376495356,0.4131123028993607 +sqrtJS,0.1,0.22130078792572022,0.3081444561481476,0.4379718750715256 +sqrtJS,0.11,0.2411379305124283,0.3316415655612946,0.46339345240592966 +sqrtJS,0.12,0.2601946280002594,0.3551857244968414,0.4895266733169557 +sqrtJS,0.13,0.2786406443119049,0.37577179908752445,0.5087037529945374 +sqrtJS,0.14,0.2961750060319901,0.39359125137329104,0.5241507070064545 +sqrtJS,0.15,0.31333486318588255,0.4112779378890991,0.5406607198715211 +sqrtJS,0.16,0.330251716375351,0.42918285250663757,0.5563057837486267 +sqrtJS,0.17,0.34669031262397765,0.4467663043737412,0.5726059519052507 +sqrtJS,0.18,0.36320728003978725,0.46412783443927763,0.589496220111847 +sqrtJS,0.19,0.37918122684955596,0.4807267391681671,0.6055341262817383 +sqrtJS,0.2,0.39338050127029417,0.4946394622325897,0.6166642975807191 +sqrtJS,0.21,0.4074707036018372,0.5085184514522553,0.6278649816513062 +sqrtJS,0.22,0.4218066644668579,0.5232259404659272,0.6389544706344604 +sqrtJS,0.23,0.4362498848438263,0.5372909414768219,0.6502852375507355 +sqrtJS,0.24,0.45056510758399965,0.551124175786972,0.6622575743198396 +sqrtJS,0.25,0.46438299119472504,0.56496861577034,0.6741052806377412 +sqrtJS,0.26,0.47699433958530424,0.5767214751243592,0.6834183979034424 +sqrtJS,0.27,0.48990054464340216,0.5884885108470916,0.6930288131237031 +sqrtJS,0.28,0.5024073088169098,0.6003974807262421,0.7026819944381715 +sqrtJS,0.29,0.5145041644573212,0.6123029220104217,0.7116441249847412 +sqrtJS,0.3,0.5269226670265198,0.6240206718444824,0.7207133269309998 +sqrtJS,0.31,0.5394867300987244,0.6356269311904907,0.7307700054645538 +sqrtJS,0.32,0.5507207024097442,0.6458725512027741,0.7394335899353027 +sqrtJS,0.33,0.5615565638542175,0.6556150567531586,0.747082168340683 +sqrtJS,0.34,0.5726638758182526,0.6653832662105561,0.7547603509426117 +sqrtJS,0.35000000000000003,0.5832657992839814,0.6751384854316712,0.7623631894588471 +sqrtJS,0.36,0.5940228672027588,0.6850643861293793,0.7701632099151612 +sqrtJS,0.37,0.6046120948791504,0.6947662425041199,0.7780641791820526 +sqrtJS,0.38,0.6149515600204468,0.7038604676723481,0.7849910805225373 +sqrtJS,0.39,0.6248047916889191,0.7123464345932007,0.7914981820583343 +sqrtJS,0.4,0.634533258676529,0.7205662369728089,0.7981813430786134 +sqrtJS,0.41000000000000003,0.6445729196071626,0.7288025903701782,0.8048498303890228 +sqrtJS,0.42,0.6539895122051239,0.7375640988349914,0.8119170551300049 +sqrtJS,0.43,0.6636344738006592,0.746227593421936,0.8185152912139894 +sqrtJS,0.44,0.6731288142204285,0.7539853453636169,0.8250248200893402 +sqrtJS,0.45,0.6820075988769532,0.7612987101078034,0.8304680418968201 +sqrtJS,0.46,0.6907292442321777,0.7686083114147186,0.8360229830741883 +sqrtJS,0.47000000000000003,0.6996124291419983,0.775828093290329,0.8417794766426087 +sqrtJS,0.48,0.7082287199497223,0.7830865216255187,0.8470603346824647 +sqrtJS,0.49,0.7169752988815308,0.7901838731765747,0.8527391023635864 +sqrtJS,0.5,0.7257071733474731,0.797585517168045,0.8583120226860047 +sqrtJS,0.51,0.7335562725067138,0.8040876352787019,0.8630223045349121 +sqrtJS,0.52,0.7415082170963287,0.8106435000896455,0.8679240298271179 +sqrtJS,0.53,0.7493822684288025,0.8170631170272827,0.8728011865615845 +sqrtJS,0.54,0.7574908215999604,0.8233809792995452,0.8776091952323914 +sqrtJS,0.55,0.7656280434131623,0.829821252822876,0.8822656393051148 +sqrtJS,0.56,0.7734538509845734,0.8360869491100311,0.8865660161972047 +sqrtJS,0.5700000000000001,0.7806796729564667,0.841989266872406,0.890981791973114 +sqrtJS,0.58,0.7878931245803833,0.8477616441249847,0.895245012998581 +sqrtJS,0.59,0.7948457033634185,0.8534522843360901,0.899298728466034 +sqrtJS,0.6,0.8018905556201934,0.8592584311962128,0.90374951004982 +sqrtJS,0.61,0.8090542590618134,0.864934024810791,0.9080019204616546 +sqrtJS,0.62,0.8164831233024598,0.8706759059429169,0.9123586592674255 +sqrtJS,0.63,0.8233440356254578,0.8760299563407898,0.91590318608284 +sqrtJS,0.64,0.8301278896331787,0.8810383892059326,0.9195370299816131 +sqrtJS,0.65,0.8364628052711487,0.8859153926372527,0.92311962723732 +sqrtJS,0.66,0.8428204205036164,0.8908948481082917,0.9265451674461365 +sqrtJS,0.67,0.8490544674396515,0.8957911968231201,0.9302592699527741 +sqrtJS,0.68,0.8554112322330475,0.9007599723339081,0.9339414570331573 +sqrtJS,0.6900000000000001,0.8616303374767305,0.9056448042392731,0.9374314184188843 +sqrtJS,0.7000000000000001,0.867506071329117,0.910114187002182,0.9404822564125062 +sqrtJS,0.71,0.8734041030406952,0.9144605398178101,0.9435326302051544 +sqrtJS,0.72,0.8792084469795227,0.9186651873588563,0.9466546597480774 +sqrtJS,0.73,0.8852027490139007,0.9230585145950317,0.9497251780033112 +sqrtJS,0.74,0.8908741686344147,0.9275643980503083,0.9528880560398102 +sqrtJS,0.75,0.8968155145645141,0.9319663047790527,0.9561091542243958 +sqrtJS,0.76,0.9022131037712097,0.9356698334217071,0.958662344455719 +sqrtJS,0.77,0.9075530467033386,0.9394511222839356,0.9613268113136291 +sqrtJS,0.78,0.9126882636547089,0.9431414198875427,0.9639042048454285 +sqrtJS,0.79,0.9177682509422302,0.9468423104286194,0.9664572861194611 +sqrtJS,0.8,0.9228397905826569,0.9506618499755859,0.9692070138454438 +sqrtJS,0.81,0.9280054275989532,0.954339977502823,0.9718747148513794 +sqrtJS,0.8200000000000001,0.9326783342361451,0.957711431980133,0.9741583812236786 +sqrtJS,0.8300000000000001,0.9370358016490936,0.9608461630344391,0.9762917485237123 +sqrtJS,0.84,0.94166756606102,0.9639502227306366,0.9783890507221221 +sqrtJS,0.85,0.9461273670196534,0.9669750392436982,0.9805065333843231 +sqrtJS,0.86,0.950419674873352,0.9701247680187225,0.9826064977645874 +sqrtJS,0.87,0.954700181722641,0.9732265996932983,0.9848827543258667 +sqrtJS,0.88,0.9588134922981263,0.975977965593338,0.9867102270126343 +sqrtJS,0.89,0.9626632099151611,0.9784288847446442,0.9880915217399597 +sqrtJS,0.9,0.9664256846904754,0.9809091985225677,0.9895272922515869 +sqrtJS,0.91,0.97026131939888,0.983419725894928,0.9910415377616882 +sqrtJS,0.92,0.9742419409751892,0.9858587121963501,0.9925662162303925 +sqrtJS,0.93,0.978017028093338,0.9882969605922699,0.994202904701233 +sqrtJS,0.9400000000000001,0.9816232523918152,0.9904669260978699,0.9957073335647584 +sqrtJS,0.9500000000000001,0.9846860337257386,0.9920557618141175,0.9964227819442749 +sqrtJS,0.96,0.9877488150596618,0.9936445975303649,0.9971382303237915 +sqrtJS,0.97,0.9908115963935852,0.9952334332466126,0.9978536787033081 +sqrtJS,0.98,0.9938743777275086,0.9968222689628601,0.9985691270828247 +sqrtJS,0.99,0.9969371590614319,0.9984111046791077,0.9992845754623414 +sqrtJS,1.0,0.9999998807907104,1.0,1.0000001192092896 diff --git a/results/head-dominance/external-caches.json b/results/head-dominance/external-caches.json new file mode 100644 index 0000000..eb1f609 --- /dev/null +++ b/results/head-dominance/external-caches.json @@ -0,0 +1,518 @@ +{ + "host": "8.211.154.134", + "checkpoint_directory": "/opt/exp-model-parameter-similarity/runs/cpu-qwen35-2b/weights", + "checkpoint_bytes": 4548221488, + "checkpoint_files": [ + { + "path": "/opt/exp-model-parameter-similarity/runs/cpu-qwen35-2b/weights/model.safetensors-00001-of-00001.safetensors", + "bytes": 4548221488, + "sha256": "aa33250c4fc64891ddfaba3a314fd9542ea371843c387178b425fbcc5ed680b1", + "matches_upstream": true + } + ], + "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "baseline_caches": [ + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/final.npy", + "bytes": 262272, + "sha256": "b8a8855e84f639683de36565455e69fc79851211df77a6041ba7a129cb238898" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-0.npz", + "bytes": 4981724, + "sha256": "62030af9dbeefd2bac83e1c76a22a10b3527b2b68208509f488e4c094ce87eff" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-1.npz", + "bytes": 4981724, + "sha256": "dfb450bf03335a3fc900d1d9093f6525eb5db46e13798b347a159ee6462ba1b8" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-10.npz", + "bytes": 4981724, + "sha256": "eef832f4c4ba777fa67de5a25dc9e15cf0b33530634707ffd5670e9c9f044ecd" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-11.npz", + "bytes": 2884572, + "sha256": "4dd4e8c8a29c698fa24e57b91b183c26bd9e05601f10589b5e82abe3b3ce3fa4" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-12.npz", + "bytes": 4981724, + "sha256": "c69a369f194f282f8fd1e60c086a2dea15fde10928b4e244342971d2fb0cc16f" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-13.npz", + "bytes": 4981724, + "sha256": "40d08716e7fea26fe8560f6ffbaf148607edd6712c897cb7cb9089bdfe71b87a" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-14.npz", + "bytes": 4981724, + "sha256": "b0fdf801c099b06098095e4d43b569376b799999d1348a25c931dbe6d68a0e1e" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-15.npz", + "bytes": 2884572, + "sha256": "d76c0568afb145d5c2b1302e821f169d64a1b06c6126bf13db4889b3c2bf76c3" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-16.npz", + "bytes": 4981724, + "sha256": "8664dc950709d968240cc2ab9c44a69dfd18227c9b1dbf0cd0f225e65c194183" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-17.npz", + "bytes": 4981724, + "sha256": "784ae9e131afbbccd2525f383f64163b0b377ac689c8ec80c6205f6a0824a5d1" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-18.npz", + "bytes": 4981724, + "sha256": "be8b8c519b3f8ef85c8bb9890a4effcc341720eb5c91039d7f4728a89a1ce81d" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-19.npz", + "bytes": 2884572, + "sha256": "c3c1d102d92eefd566ac6cb36c348942440d3b06203b4da1ee6737923eaeecd9" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-2.npz", + "bytes": 4981724, + "sha256": "460da9b3dfe4e8dd0f5d654a5ff42948c4964d8bb808f3fb453d4acca3c2b171" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-20.npz", + "bytes": 4981724, + "sha256": "dea32d068464045dd5b43e4e98362eab3790ad2720a19f5b26bd1be20a8a9052" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-21.npz", + "bytes": 4981724, + "sha256": "41a1cce4928ccc7e84184676f8a32637137103f56e0892987bc6330ab8f56da8" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-22.npz", + "bytes": 4981724, + "sha256": "c507ce7c243dd74fa58bd77e692e23e13f12c39f45e4105a1609edfeaa7155b8" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-23.npz", + "bytes": 2884572, + "sha256": "8a99178b0c6b0fc83210c67fe340b6d02d30fe47b21b2159e5c332f7fdb0b767" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-3.npz", + "bytes": 2884572, + "sha256": "7bdb96ef838151402a9742039f2fe003ecfc0754a5edd6e430cf29810bdf47a4" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-4.npz", + "bytes": 4981724, + "sha256": "3cd67d5f339766701ed1cb1fd448424510a7cae75ff883446983a8ce1ed1480c" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-5.npz", + "bytes": 4981724, + "sha256": "19db87e1e97fd14c576c711636990e14fb18df36df9c09f21178272c411e245b" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-6.npz", + "bytes": 4981724, + "sha256": "20b0598f70ac4db3fc86c61fe1e16d90efb1136a1f7ef3fb2f2c0c453df67499" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-7.npz", + "bytes": 2884572, + "sha256": "de0fb2470d8aa8ec058a93c77a4f608be5d93da034bbf06dccb879d157e2ea92" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-8.npz", + "bytes": 4981724, + "sha256": "bc6fe847346842fbebf0fcb43494f65b0c02e283c8bc4c268ac4ca31ca888f42" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-1/layer-9.npz", + "bytes": 4981724, + "sha256": "660f7b2af375f2aca2d1db04e15fa38d996f1ec64d95cf55a1f757d2df13bd6f" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/final.npy", + "bytes": 262272, + "sha256": "2a5b66c0c5413ce20eb5dae8537267d85b74f4c8d33c9cc628c545bf14dbfb5a" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-0.npz", + "bytes": 4981724, + "sha256": "5081be33b7c04e78af91b1252f37db3ae8820da9cbef069494eb682201f31a67" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-1.npz", + "bytes": 4981724, + "sha256": "4b6f7a9029c648ea77ae453a904a272ab03ee9ac7d943673615614d806c9aaec" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-10.npz", + "bytes": 4981724, + "sha256": "4d4dbfaf77a890f1a3559b6a48904f03d06c0b9605fc6e925b000fb2b4386c40" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-11.npz", + "bytes": 2884572, + "sha256": "4ca936624bd8a394a0d1b8104c8bf584fff43ee4c79ba328b5446126702c587c" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-12.npz", + "bytes": 4981724, + "sha256": "07441023bb95d76aa6de6b7fcf2e26db30fe9b0a367b0aa1dc5bd4eee1ea0ca9" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-13.npz", + "bytes": 4981724, + "sha256": "0b81bbe8aea3ae70b7148a99ef49d52eb3dc024500c87984061059571e781cfb" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-14.npz", + "bytes": 4981724, + "sha256": "f002001cb42ad714162fbe5d9b0b75c2b0c2872be995b72cf1beb34498249f88" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-15.npz", + "bytes": 2884572, + "sha256": "1badaee06895681b6099885fbefe5f34480a0e707ec16587d83e7f90e43b4566" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-16.npz", + "bytes": 4981724, + "sha256": "ab792e9dc16abcbbbad342e2f736fa76461f444f7d63bc5b52728b6ff7db06bb" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-17.npz", + "bytes": 4981724, + "sha256": "ca74afba19125bf5d053c8eef0714fceaade023d5f494ac486d7535026c74ae3" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-18.npz", + "bytes": 4981724, + "sha256": "5776b9a0638ef4188d00bf46fa0de0f82dd678b99858d0d62cbfdf7613bdc9bc" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-19.npz", + "bytes": 2884572, + "sha256": "e2fc9bd457685b476bfbd1a246221dd8f8c4a12fa61fbad5d46faa9495b45c89" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-2.npz", + "bytes": 4981724, + "sha256": "565c4bd4f2e2aa939c20fd4d171ff2025623d9f7cae9eeacc480100af398bf0a" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-20.npz", + "bytes": 4981724, + "sha256": "1c92cd35fddbbf518f834a43ddbe2e365f19701a5f356875854c216735491e4a" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-21.npz", + "bytes": 4981724, + "sha256": "5f483ccdd61214f870622cb626e9ddc57efbae286fcdf23819e08c889a333eb6" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-22.npz", + "bytes": 4981724, + "sha256": "9af76327c2e460b65dc97db03ee86e89421836da3ea7a74da398eb395c94fe00" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-23.npz", + "bytes": 2884572, + "sha256": "02a58553f166984514554c5391ae08cbd5fbaea92734ebfc8afcc56acc2bcb4f" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-3.npz", + "bytes": 2884572, + "sha256": "ba7d9e7af38f3e4ae9079295d2de138c35ea0119c8ef145cc920b951817af7a1" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-4.npz", + "bytes": 4981724, + "sha256": "7af6705fdd8a3641d6edfc7d49390612658e5652167c0cf14051b91927999727" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-5.npz", + "bytes": 4981724, + "sha256": "df0e27d1874d306bd7ecd7aaf411d38aa37a1dac9e9c6f3c67ddd178e333de7d" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-6.npz", + "bytes": 4981724, + "sha256": "41b569309c746c89395aca0440479169d94032b6eda42c52f012ccdedeac4d6d" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-7.npz", + "bytes": 2884572, + "sha256": "45c43e3d147dbdd5829d90728d26aabd6816ee1c9525e5e8fdbca66c36619886" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-8.npz", + "bytes": 4981724, + "sha256": "3b0965b9797034b300aa7c0533ea5c2502a29981f0811424ea4330f2bddea083" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-en-2/layer-9.npz", + "bytes": 4981724, + "sha256": "545ca6d0dff4b00b950752821def6839fc9cd2f610801942b69deb8528b77457" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/final.npy", + "bytes": 262272, + "sha256": "922629c653aafb8dd7a83ee78f5c6e112d05da35e18de3d4a2e4f6bf68d53365" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-0.npz", + "bytes": 4981724, + "sha256": "f5f7072ea222c07877b4d07a056cc86340b341c22928296435763f651a038ced" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-1.npz", + "bytes": 4981724, + "sha256": "5901023b2d7a32a47ad792d5a667b26d23ef669145ec02dd820dac92f20af0f4" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-10.npz", + "bytes": 4981724, + "sha256": "a9b5967097b677d7759846d16206755f758fef65d0272db0e18396b0b1520a6f" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-11.npz", + "bytes": 2884572, + "sha256": "f70e8374779bc5c6ab5a6533f38b3680808dae7c763a158fd7457c43841613d6" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-12.npz", + "bytes": 4981724, + "sha256": "2ff252808ee96e32a5d76879e6d2d5234df5fdfd7e340a67063b7ef4343ec9cb" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-13.npz", + "bytes": 4981724, + "sha256": "efe868feb83a58ae5cba4296021ac73116e08c9c3471d53fa9919a2e971e6300" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-14.npz", + "bytes": 4981724, + "sha256": "d6945c1fe0a88f15fdf14a19c957ea2d7bd08a131985cce44b1dc6c74eaef0f3" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-15.npz", + "bytes": 2884572, + "sha256": "11dd045d2e5f830431b737ebb6f727e3f7002a2d6705518b273f518ae943b9f8" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-16.npz", + "bytes": 4981724, + "sha256": "16842459edbf8b86e6757c3ac2ae0391e40eb9a3b72f4fd37185ea217e16bff1" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-17.npz", + "bytes": 4981724, + "sha256": "b56ebceaba8ae37235dc8971d23d5dc7687f886110041aac22b902488c820d8b" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-18.npz", + "bytes": 4981724, + "sha256": "b0ee4a9555f0cabd9e9675f84d7c6ae0e05cd6f69fdd04260003a17dc84aa8da" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-19.npz", + "bytes": 2884572, + "sha256": "8246aab24bcd261a83f0c5e2d5a67cc5ed9aa4ef029c53f52828b94f1a64a0e4" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-2.npz", + "bytes": 4981724, + "sha256": "cdf294815a9c07887146655086202690c40cef6a92d6d9ff2d9f3c373382e66c" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-20.npz", + "bytes": 4981724, + "sha256": "1964019f2a7e73bcff89eea4dd0d9a82971ab0d0b2f6e3ccba7a5f7ef5fbae01" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-21.npz", + "bytes": 4981724, + "sha256": "bb0c8815c4bfbdb5c8940f3d6260dc7313c22d7406652ba5f09068661b2937b6" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-22.npz", + "bytes": 4981724, + "sha256": "50a03d0a1974a5134386405bd85e6d751e361e352aad9ea6f90629a9be001e89" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-23.npz", + "bytes": 2884572, + "sha256": "0cd2412706e8fc0f332d5d2bc201fdb63e03b96ce1d8c13748ee4e62418a35a5" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-3.npz", + "bytes": 2884572, + "sha256": "c5e81933e9f1f75dd98b019db9e876344edfb6365f44897c5db52601f3d8919c" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-4.npz", + "bytes": 4981724, + "sha256": "e731d7e45a3a0060edaef15c538e076db05a54de658db001a82c6bfaaf1a4bd1" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-5.npz", + "bytes": 4981724, + "sha256": "3203a9bd4353033f42d765d08769ce677c67ea2d8fb89b236d00b259ad86a6f5" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-6.npz", + "bytes": 4981724, + "sha256": "22590aed426ae7324831082ae48cbbdb047fe6ba487656d4918b25e1ccd2273a" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-7.npz", + "bytes": 2884572, + "sha256": "da6a7c4cbcf4eaec58a291a031cae79be427bd53827451672bd37cbe27b0f753" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-8.npz", + "bytes": 4981724, + "sha256": "4ec711d85d8bd75b45310e6fb4b3428090d423fbfb12015694aad5220a7a0d66" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-1/layer-9.npz", + "bytes": 4981724, + "sha256": "a8a0a3925e43d0d98aac56218ccb338a1a9b53b1e7ac201ade3a4be649eb5be9" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/final.npy", + "bytes": 262272, + "sha256": "bdacc87b882ec4dbf1da12d14bd4c01ddb269c67222afbf8438db2700994fbe2" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-0.npz", + "bytes": 4981724, + "sha256": "8a3b1f5d2c83e20b87f8361ff51c8a4d04b542e69394f1e3328a31752662b2a5" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-1.npz", + "bytes": 4981724, + "sha256": "02e3448e5c68a2520aeab7302e3acd86e403684fc2822dff55c88e549453df72" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-10.npz", + "bytes": 4981724, + "sha256": "ef035d3d3bbf850285b22706e7944a50c3cf5178a4c1c3a9dadab3a4dec57ad8" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-11.npz", + "bytes": 2884572, + "sha256": "b1d60e33b7407ff7512c7685f15099e93678b00cb37ed80dcefb0a26ddf452aa" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-12.npz", + "bytes": 4981724, + "sha256": "1d238079550dbd6c0a72e470d9127059f25abddce6303abee6952d7961361644" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-13.npz", + "bytes": 4981724, + "sha256": "7d1a4f9b9f1388dc8464d5c11c4cb597abc722efc8a176e28002561169e68aa4" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-14.npz", + "bytes": 4981724, + "sha256": "eb74f49de7fdfcfe782016d1787669b2ea690514ec30d7e0650fc3c732168181" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-15.npz", + "bytes": 2884572, + "sha256": "4aefbeec78ebd3a59b63b4d2243d560673e2f32c54d700fe15529fe98d2f7157" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-16.npz", + "bytes": 4981724, + "sha256": "1480eadf962298ee560d5de5a44c753403b8704c26de06d9c25eb3537ca4b56f" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-17.npz", + "bytes": 4981724, + "sha256": "c1ad8f0695eb640873e58e2c464fb4d6c3dc0825f8d382abda561ceeb8851853" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-18.npz", + "bytes": 4981724, + "sha256": "453e6c478168f10d4fb5a734c01f6a415e0756d377fb8673976293b40ad9e4ac" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-19.npz", + "bytes": 2884572, + "sha256": "4010a06526186c4d058f2860d8bee3bfb84db0cbeaa2134fb545903ba0fae078" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-2.npz", + "bytes": 4981724, + "sha256": "af7441b1f57b72fa38b8a8c3327836b866ac33cb52dc9bd3b5d70dafb9bcc0e2" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-20.npz", + "bytes": 4981724, + "sha256": "1c5c0b9a7d6c32d2806f3fb194345d41973472fe50e2b1b986dc89bcf736c873" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-21.npz", + "bytes": 4981724, + "sha256": "9a5412f548dcc485d05e1e862eade8d0d5cbff7f773bf0c2735ae369e6e1c972" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-22.npz", + "bytes": 4981724, + "sha256": "b0951d76dab81d9859d16e7389bd6d35803058d2595b84a337d29e7a8155ccaf" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-23.npz", + "bytes": 2884572, + "sha256": "2ddfc44d49e79761312446e359cb601c53c98077174042ec2230fc2c0499316f" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-3.npz", + "bytes": 2884572, + "sha256": "521df2e9dc28cbcc04d174e7aa8e1b853752b661048fa9f65990893ff81d2a4b" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-4.npz", + "bytes": 4981724, + "sha256": "bedfbe9fa97663c32a0d80a9198909e9caaf6eaf64b2f2d1bd6b6a6ec7a478b8" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-5.npz", + "bytes": 4981724, + "sha256": "bd7d5eec27dfcf2031f07011704a9ad91a4668b4b063f17fba0c2603cd774c71" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-6.npz", + "bytes": 4981724, + "sha256": "626bda0c0d1b09e6a35a454f2ed6739ed70a58717f9c67ed79faa48027159ecd" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-7.npz", + "bytes": 2884572, + "sha256": "9c7092c40d7c4eb83bbce9acb4faf810e24bc79515c6e2423ca18c9ff7c87fe6" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-8.npz", + "bytes": 4981724, + "sha256": "430955c134cf3e486aa0fa8035a664d6bad40f28adfb3e62cde2ff6218c9f286" + }, + { + "path": "/opt/exp-model-parameter-similarity/runs/head-dominance/cache/fresh-zh-2/layer-9.npz", + "bytes": 4981724, + "sha256": "b38b92b9224a7d1c31b3b451e9af7774c8de3a2dd8f71567cf3590098cc2dae4" + } + ], + "automatic_deletion": false, + "prior_caches_untouched": true +} diff --git a/results/head-dominance/figures/continuous_dominance.pdf b/results/head-dominance/figures/continuous_dominance.pdf new file mode 100644 index 0000000..87024ef Binary files /dev/null and b/results/head-dominance/figures/continuous_dominance.pdf differ diff --git a/results/head-dominance/figures/continuous_dominance.png b/results/head-dominance/figures/continuous_dominance.png new file mode 100644 index 0000000..99a7fdf Binary files /dev/null and b/results/head-dominance/figures/continuous_dominance.png differ diff --git a/results/head-dominance/figures/continuous_dominance.svg b/results/head-dominance/figures/continuous_dominance.svg new file mode 100644 index 0000000..1f6e6c3 --- /dev/null +++ b/results/head-dominance/figures/continuous_dominance.svg @@ -0,0 +1,2897 @@ + + + + + + + + 2026-11-10T22:31:08.061287 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/results/head-dominance/figures/joint_removal_effects.pdf b/results/head-dominance/figures/joint_removal_effects.pdf new file mode 100644 index 0000000..65c5f34 Binary files /dev/null and b/results/head-dominance/figures/joint_removal_effects.pdf differ diff --git a/results/head-dominance/figures/joint_removal_effects.png b/results/head-dominance/figures/joint_removal_effects.png new file mode 100644 index 0000000..526b9c2 Binary files /dev/null and b/results/head-dominance/figures/joint_removal_effects.png differ diff --git a/results/head-dominance/figures/joint_removal_effects.svg b/results/head-dominance/figures/joint_removal_effects.svg new file mode 100644 index 0000000..32b2ef6 --- /dev/null +++ b/results/head-dominance/figures/joint_removal_effects.svg @@ -0,0 +1,2631 @@ + + + + + + + + 2026-11-10T22:31:17.017973 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/results/head-dominance/figures/ranked_influence_distribution.pdf b/results/head-dominance/figures/ranked_influence_distribution.pdf new file mode 100644 index 0000000..ce08efd Binary files /dev/null and b/results/head-dominance/figures/ranked_influence_distribution.pdf differ diff --git a/results/head-dominance/figures/ranked_influence_distribution.png b/results/head-dominance/figures/ranked_influence_distribution.png new file mode 100644 index 0000000..96cf845 Binary files /dev/null and b/results/head-dominance/figures/ranked_influence_distribution.png differ diff --git a/results/head-dominance/figures/ranked_influence_distribution.svg b/results/head-dominance/figures/ranked_influence_distribution.svg new file mode 100644 index 0000000..6bff10c --- /dev/null +++ b/results/head-dominance/figures/ranked_influence_distribution.svg @@ -0,0 +1,5717 @@ + + + + + + + + 2026-11-10T22:31:13.169784 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/results/head-dominance/figures/retrieval_scale_sensitivity.pdf b/results/head-dominance/figures/retrieval_scale_sensitivity.pdf new file mode 100644 index 0000000..9da7300 Binary files /dev/null and b/results/head-dominance/figures/retrieval_scale_sensitivity.pdf differ diff --git a/results/head-dominance/figures/retrieval_scale_sensitivity.png b/results/head-dominance/figures/retrieval_scale_sensitivity.png new file mode 100644 index 0000000..1b51c1f Binary files /dev/null and b/results/head-dominance/figures/retrieval_scale_sensitivity.png differ diff --git a/results/head-dominance/figures/retrieval_scale_sensitivity.svg b/results/head-dominance/figures/retrieval_scale_sensitivity.svg new file mode 100644 index 0000000..ce5b9d9 --- /dev/null +++ b/results/head-dominance/figures/retrieval_scale_sensitivity.svg @@ -0,0 +1,4105 @@ + + + + + + + + 2026-11-10T22:31:19.773050 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/results/head-dominance/final-tests.txt b/results/head-dominance/final-tests.txt new file mode 100644 index 0000000..202f31e --- /dev/null +++ b/results/head-dominance/final-tests.txt @@ -0,0 +1,42 @@ +test_3d_reader_and_exact_block (tests.test_pilot.PilotTests.test_3d_reader_and_exact_block) ... ok +test_bfloat16 (tests.test_pilot.PilotTests.test_bfloat16) ... ok +test_directional_replacement_error_and_scaling (tests.test_pilot.PilotTests.test_directional_replacement_error_and_scaling) ... ok +test_end_to_end_synthetic_groups_and_savings (tests.test_pilot.PilotTests.test_end_to_end_synthetic_groups_and_savings) ... Inventory: 4 tensors, 33792 bytes +tensor: sampling 4 objects +block: sampling 22 objects +block: verified 100/109 +{"model": "synthetic", "revision": "test", "tensor_count": 4, "parameter_count": 8448, "model_weight_bytes": 33792, "ndim_counts": {"2": 2, "3": 2}, "elapsed_seconds": 0.09181240666657686, "analysis_budget_seconds": 60, "peak_rss_kib": 55188, "python": "3.14.4", "numpy": "2.5.4", "seed": 20261109, "limitations": ["No inference or functional equivalence measured", "No exhaustive pair search", "Block search uses per-role/shape/scale reservoirs of 256", "Different block scales overlap; savings must not be added", "Block savings count only verified disjoint sampled blocks, no extrapolation", "Spectral probes use aligned bounded subvolumes, not whole tensors", "Shared encoding is theoretical, not a runnable compressed checkpoint", "No cross-role grouping, neuron permutation alignment, or training"], "completed": true} +ok +test_no_transitive_chaining (tests.test_pilot.PilotTests.test_no_transitive_chaining) ... ok +test_ranged_download_preserves_prefix (tests.test_pilot.PilotTests.test_ranged_download_preserves_prefix) ... Weight download 36/36 bytes (100.0%) +ok +test_causal_convolution_and_norm_boundary (tests.test_functional.FunctionalTests.test_causal_convolution_and_norm_boundary) ... ok +test_causality_and_identical_function_metrics (tests.test_functional.FunctionalTests.test_causality_and_identical_function_metrics) ... ok +test_neuron_function_metrics_and_targeted_replacement (tests.test_functional.FunctionalTests.test_neuron_function_metrics_and_targeted_replacement) ... ok +test_nonrotary_gain_identity_and_rotary_negative_control (tests.test_functional.FunctionalTests.test_nonrotary_gain_identity_and_rotary_negative_control) ... ok +test_streamed_projection_embedding_and_output_head (tests.test_functional.FunctionalTests.test_streamed_projection_embedding_and_output_head) ... ok +test_uniform_rmsnorm_function_bound_is_tight (tests.test_functional.FunctionalTests.test_uniform_rmsnorm_function_bound_is_tight) ... ok +test_assignment_against_exhaustive_and_inverse (tests.test_crossqkv.CrossQKVTests.test_assignment_against_exhaustive_and_inverse) ... ok +test_cached_mode_rejects_future_state_reuse (tests.test_crossqkv.CrossQKVTests.test_cached_mode_rejects_future_state_reuse) ... ok +test_controls_cover_multiple_projection_layers (tests.test_crossqkv.CrossQKVTests.test_controls_cover_multiple_projection_layers) ... ok +test_head_matrix_metrics_and_gram_are_diagnostic (tests.test_crossqkv.CrossQKVTests.test_head_matrix_metrics_and_gram_are_diagnostic) ... ok +test_probability_identity_and_nontrivial_queries (tests.test_crossqkv.CrossQKVTests.test_probability_identity_and_nontrivial_queries) ... ok +test_profile_matches_canonical_and_head_sum (tests.test_crossqkv.CrossQKVTests.test_profile_matches_canonical_and_head_sum) ... ok +test_recipient_value_gate_output_and_gqa_group_stay_fixed (tests.test_crossqkv.CrossQKVTests.test_recipient_value_gate_output_and_gqa_group_stay_fixed) ... ok +test_topk_tail_rank_can_change_with_tiny_probability_difference (tests.test_crossqkv.CrossQKVTests.test_topk_tail_rank_can_change_with_tiny_probability_difference) ... ok +test_continuous_limits_and_undefined_zero (tests.test_dominance.DominanceTests.test_continuous_limits_and_undefined_zero) ... ok +test_joint_recomputation_null_and_saved_raw (tests.test_dominance.DominanceTests.test_joint_recomputation_null_and_saved_raw) ... baseline layer 0 +baseline layer 1 +ok +test_js_identity_and_disjoint (tests.test_dominance.DominanceTests.test_js_identity_and_disjoint) ... ok +test_mask_ranks_vary_by_token_and_recipient (tests.test_dominance.DominanceTests.test_mask_ranks_vary_by_token_and_recipient) ... ok +test_mixed_layer_forward_and_effective_qkv (tests.test_dominance.DominanceTests.test_mixed_layer_forward_and_effective_qkv) ... baseline layer 0 +baseline layer 1 +baseline layer 0 +baseline layer 1 +ok + +---------------------------------------------------------------------- +Ran 25 tests in 22.226s + +OK diff --git a/results/head-dominance/head-sum-controls.json b/results/head-dominance/head-sum-controls.json new file mode 100644 index 0000000..2809e5e --- /dev/null +++ b/results/head-dominance/head-sum-controls.json @@ -0,0 +1,482 @@ +[ + { + "sample": "fresh-zh-1", + "layer": 0, + "relative_l2": 3.063783194036455e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 1, + "relative_l2": 3.050940047416459e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 2, + "relative_l2": 2.8865816853682123e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 3, + "relative_l2": 4.5566014249534336e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 4, + "relative_l2": 3.375571321543082e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 5, + "relative_l2": 3.760860919744642e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 6, + "relative_l2": 4.060386356602171e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 7, + "relative_l2": 6.142276712055951e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 8, + "relative_l2": 4.260616376282382e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 9, + "relative_l2": 4.05921646623785e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 10, + "relative_l2": 4.0162681354075384e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 11, + "relative_l2": 6.346386287817742e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 12, + "relative_l2": 3.6290315220180745e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 13, + "relative_l2": 3.832751858626575e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 14, + "relative_l2": 3.403046318084744e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 15, + "relative_l2": 2.6412964511173763e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 16, + "relative_l2": 3.35423943710327e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 17, + "relative_l2": 3.629367703764049e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 18, + "relative_l2": 3.382336050867843e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 19, + "relative_l2": 4.111871649115565e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 20, + "relative_l2": 2.6695487574151434e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 21, + "relative_l2": 3.5647735014205986e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 22, + "relative_l2": 3.709029871835037e-07 + }, + { + "sample": "fresh-zh-1", + "layer": 23, + "relative_l2": 4.095975758975568e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 0, + "relative_l2": 3.1054236333332787e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 1, + "relative_l2": 3.2344941513198663e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 2, + "relative_l2": 2.805990410561153e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 3, + "relative_l2": 4.7011349205079565e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 4, + "relative_l2": 3.806898423128068e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 5, + "relative_l2": 4.004418415748534e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 6, + "relative_l2": 4.3436543195058997e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 7, + "relative_l2": 6.202020302520739e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 8, + "relative_l2": 4.1713519847643574e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 9, + "relative_l2": 3.968074235546914e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 10, + "relative_l2": 4.0252648531522866e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 11, + "relative_l2": 6.392567575363303e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 12, + "relative_l2": 3.699184743483327e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 13, + "relative_l2": 3.8435361174879136e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 14, + "relative_l2": 3.5000931282959163e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 15, + "relative_l2": 2.7679069331185467e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 16, + "relative_l2": 3.471358657198979e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 17, + "relative_l2": 3.951482219285595e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 18, + "relative_l2": 3.5732359293583374e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 19, + "relative_l2": 4.2266102278121267e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 20, + "relative_l2": 2.900666494300031e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 21, + "relative_l2": 3.742688455833942e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 22, + "relative_l2": 3.772310984201327e-07 + }, + { + "sample": "fresh-zh-2", + "layer": 23, + "relative_l2": 4.217492862855182e-07 + }, + { + "sample": "fresh-en-1", + "layer": 0, + "relative_l2": 2.855953547442806e-07 + }, + { + "sample": "fresh-en-1", + "layer": 1, + "relative_l2": 2.9424892641997365e-07 + }, + { + "sample": "fresh-en-1", + "layer": 2, + "relative_l2": 3.4201968670138326e-07 + }, + { + "sample": "fresh-en-1", + "layer": 3, + "relative_l2": 4.623465157836892e-07 + }, + { + "sample": "fresh-en-1", + "layer": 4, + "relative_l2": 3.561553796976913e-07 + }, + { + "sample": "fresh-en-1", + "layer": 5, + "relative_l2": 3.8063468995667835e-07 + }, + { + "sample": "fresh-en-1", + "layer": 6, + "relative_l2": 4.275793736362091e-07 + }, + { + "sample": "fresh-en-1", + "layer": 7, + "relative_l2": 6.105892642482881e-07 + }, + { + "sample": "fresh-en-1", + "layer": 8, + "relative_l2": 4.223430107585696e-07 + }, + { + "sample": "fresh-en-1", + "layer": 9, + "relative_l2": 3.989933081474535e-07 + }, + { + "sample": "fresh-en-1", + "layer": 10, + "relative_l2": 3.9231650725660523e-07 + }, + { + "sample": "fresh-en-1", + "layer": 11, + "relative_l2": 6.522987261303035e-07 + }, + { + "sample": "fresh-en-1", + "layer": 12, + "relative_l2": 3.6140553557673553e-07 + }, + { + "sample": "fresh-en-1", + "layer": 13, + "relative_l2": 4.0705416274232034e-07 + }, + { + "sample": "fresh-en-1", + "layer": 14, + "relative_l2": 3.478787357234133e-07 + }, + { + "sample": "fresh-en-1", + "layer": 15, + "relative_l2": 2.6142252259772924e-07 + }, + { + "sample": "fresh-en-1", + "layer": 16, + "relative_l2": 3.5778068114215066e-07 + }, + { + "sample": "fresh-en-1", + "layer": 17, + "relative_l2": 3.8764969214608894e-07 + }, + { + "sample": "fresh-en-1", + "layer": 18, + "relative_l2": 3.5971881224092595e-07 + }, + { + "sample": "fresh-en-1", + "layer": 19, + "relative_l2": 4.0329464698542476e-07 + }, + { + "sample": "fresh-en-1", + "layer": 20, + "relative_l2": 3.0784999968330634e-07 + }, + { + "sample": "fresh-en-1", + "layer": 21, + "relative_l2": 3.4511593739527337e-07 + }, + { + "sample": "fresh-en-1", + "layer": 22, + "relative_l2": 3.741078231288514e-07 + }, + { + "sample": "fresh-en-1", + "layer": 23, + "relative_l2": 3.826857136572196e-07 + }, + { + "sample": "fresh-en-2", + "layer": 0, + "relative_l2": 2.731862254495047e-07 + }, + { + "sample": "fresh-en-2", + "layer": 1, + "relative_l2": 3.318794528269955e-07 + }, + { + "sample": "fresh-en-2", + "layer": 2, + "relative_l2": 3.118584003777839e-07 + }, + { + "sample": "fresh-en-2", + "layer": 3, + "relative_l2": 4.51243844395859e-07 + }, + { + "sample": "fresh-en-2", + "layer": 4, + "relative_l2": 3.549449011584008e-07 + }, + { + "sample": "fresh-en-2", + "layer": 5, + "relative_l2": 3.7640972731258426e-07 + }, + { + "sample": "fresh-en-2", + "layer": 6, + "relative_l2": 4.232629595364006e-07 + }, + { + "sample": "fresh-en-2", + "layer": 7, + "relative_l2": 6.014577598645773e-07 + }, + { + "sample": "fresh-en-2", + "layer": 8, + "relative_l2": 4.247301141043684e-07 + }, + { + "sample": "fresh-en-2", + "layer": 9, + "relative_l2": 4.043636615769786e-07 + }, + { + "sample": "fresh-en-2", + "layer": 10, + "relative_l2": 4.0446221605246967e-07 + }, + { + "sample": "fresh-en-2", + "layer": 11, + "relative_l2": 6.281014678655475e-07 + }, + { + "sample": "fresh-en-2", + "layer": 12, + "relative_l2": 3.9070227166187065e-07 + }, + { + "sample": "fresh-en-2", + "layer": 13, + "relative_l2": 4.029177182015629e-07 + }, + { + "sample": "fresh-en-2", + "layer": 14, + "relative_l2": 3.4864487021346635e-07 + }, + { + "sample": "fresh-en-2", + "layer": 15, + "relative_l2": 2.5625119980181207e-07 + }, + { + "sample": "fresh-en-2", + "layer": 16, + "relative_l2": 3.524751982123894e-07 + }, + { + "sample": "fresh-en-2", + "layer": 17, + "relative_l2": 3.8012783383650425e-07 + }, + { + "sample": "fresh-en-2", + "layer": 18, + "relative_l2": 3.7332363586793186e-07 + }, + { + "sample": "fresh-en-2", + "layer": 19, + "relative_l2": 4.2440165172713985e-07 + }, + { + "sample": "fresh-en-2", + "layer": 20, + "relative_l2": 3.0638678945010683e-07 + }, + { + "sample": "fresh-en-2", + "layer": 21, + "relative_l2": 3.48562079754112e-07 + }, + { + "sample": "fresh-en-2", + "layer": 22, + "relative_l2": 3.7450658917225235e-07 + }, + { + "sample": "fresh-en-2", + "layer": 23, + "relative_l2": 3.7450176622059096e-07 + } +] diff --git a/results/head-dominance/joint-by-link.csv b/results/head-dominance/joint-by-link.csv new file mode 100644 index 0000000..89484e4 --- /dev/null +++ b/results/head-dominance/joint-by-link.csv @@ -0,0 +1,1665 @@ +sample,source,recipient,metric,removed,joint_effect_count,joint_effect_p10,joint_effect_median,joint_effect_p90,joint_effect_mean,sum_individual_count,sum_individual_p10,sum_individual_median,sum_individual_p90,sum_individual_mean +fresh-zh-1,0,1,Q,weak_half,400,0.2008264407515526,0.3334670960903168,0.4958236008882523,0.34076051905751226,400,0.4859659373760224,0.7440362572669983,1.0349951386451721,0.7626679044216871 +fresh-zh-1,0,1,Q,strong_half,400,0.7453873157501221,1.0975597500801086,1.3908109784126281,1.0837196806818248,400,1.7807375907897949,2.7196731567382812,3.719470500946045,2.751625395715237 +fresh-zh-1,0,1,Q,random_half,400,0.4882308393716812,1.0184450149536133,1.407813024520874,0.979760735668242,400,0.9431869566440583,1.651898205280304,2.6601041316986085,1.7278785783052444 +fresh-zh-1,0,1,Q,all_heads,400,0.9995198369026185,1.2742709517478943,1.4928893208503724,1.257547627389431,400,2.4291015148162844,3.5201497077941895,4.555028486251832,3.5142933094501494 +fresh-zh-1,0,1,K,weak_half,400,0.25027612447738645,0.38210147619247437,0.584058403968811,0.40587454933673145,400,0.5675718903541564,0.8254277408123016,1.2087530612945558,0.853133482709527 +fresh-zh-1,0,1,K,strong_half,400,0.7469826638698578,1.0280970335006714,1.3284502983093263,1.0319453468173743,400,1.9396398782730102,2.5797523260116577,3.574095940589905,2.6945008167624476 +fresh-zh-1,0,1,K,random_half,400,0.605703216791153,0.91021528840065,1.2458263754844667,0.9210443340986967,400,1.116757595539093,1.6681658625602722,2.5559180974960327,1.7548867759108544 +fresh-zh-1,0,1,K,all_heads,400,1.067708468437195,1.2824383974075317,1.4543998718261721,1.265971492677927,400,2.6258231163024903,3.4643481969833374,4.545622396469116,3.54763429492712 +fresh-zh-1,0,1,V,weak_half,400,0.14895713031291966,0.31546591222286224,0.6157237470149994,0.36670552714727817,400,0.35644741654396056,0.6959381699562073,1.0560676336288453,0.7181011345982552 +fresh-zh-1,0,1,V,strong_half,400,0.46852283775806425,0.8431256115436554,1.5678224563598633,0.9399358623102307,400,1.264595413208008,2.248305082321167,4.21556830406189,2.596791000217199 +fresh-zh-1,0,1,V,random_half,400,0.3486655354499817,0.7840900123119354,1.7785831928253177,0.9870144194178283,400,0.7437542021274567,1.4282178282737732,2.746462202072145,1.6596856532245874 +fresh-zh-1,0,1,V,all_heads,400,0.6816939592361451,1.0619012117385864,1.6543446063995362,1.1273160568997265,400,1.661899483203888,2.9560275077819824,5.0967691421508805,3.3148921114206313 +fresh-zh-1,0,1,QKV,weak_half,400,0.2623360425233841,0.37220193445682526,0.5257133841514587,0.3884131891280413,400,0.6090817153453827,0.79803666472435,1.0377546548843384,0.8178685554116965 +fresh-zh-1,0,1,QKV,strong_half,400,0.7960836172103882,1.018787920475006,1.3160068631172182,1.047460880726576,400,2.1024187803268433,2.713298797607422,3.7205054759979257,2.8510900834202766 +fresh-zh-1,0,1,QKV,random_half,400,0.6320619761943818,0.9534361958503723,1.3505552887916568,1.0151568105071782,400,1.1541799306869507,1.7030929327011108,2.633734917640686,1.8170585039258003 +fresh-zh-1,0,1,QKV,all_heads,400,1.0367567896842957,1.2274160981178284,1.4590619444847106,1.2396013320982455,400,2.780677604675293,3.527419328689575,4.623041152954102,3.668958649635315 +fresh-zh-1,1,2,Q,weak_half,400,0.03155932053923607,0.06444158777594566,0.1082404971122742,0.06897579949581996,400,0.07613841444253922,0.14699863642454147,0.23956551849842073,0.1547582809627056 +fresh-zh-1,1,2,Q,strong_half,400,0.18894434422254563,0.36650605499744415,0.604983013868332,0.38401546245440843,400,0.40905213654041295,0.8548521995544434,1.3071335434913636,0.8611730960384011 +fresh-zh-1,1,2,Q,random_half,400,0.1108572743833065,0.226791612803936,0.3800503104925157,0.23927256184630097,400,0.2336871862411499,0.47147881984710693,0.7881120920181276,0.5016119395382702 +fresh-zh-1,1,2,Q,all_heads,400,0.19902447313070298,0.3936949819326401,0.6310463190078736,0.4032550751976669,400,0.49195235371589663,1.013447344303131,1.5085470914840697,1.0159313766658307 +fresh-zh-1,1,2,K,weak_half,400,0.046294156834483145,0.07594529911875725,0.13886554837226872,0.08780013904441147,400,0.10614747181534769,0.17040395736694336,0.28339419066905974,0.18473220139741897 +fresh-zh-1,1,2,K,strong_half,400,0.2527156352996826,0.39471787214279175,0.6004869639873505,0.4146555851027369,400,0.6010036945343018,0.8860439658164978,1.3159768342971803,0.9252161803096532 +fresh-zh-1,1,2,K,random_half,400,0.15547478497028352,0.253771036863327,0.41864116787910466,0.2748459529224783,400,0.3095446139574051,0.5251365303993225,0.8514812231063843,0.5517403537034988 +fresh-zh-1,1,2,K,all_heads,400,0.2693099319934845,0.42332057654857635,0.6184103012084962,0.4363914165645838,400,0.7223121941089631,1.0673685669898987,1.5319440245628357,1.109948377609253 +fresh-zh-1,1,2,V,weak_half,400,0.027415544353425504,0.06430166959762573,0.1410903006792069,0.07556237921584398,400,0.06414011716842652,0.13570336252450943,0.26234134137630466,0.15288431376218795 +fresh-zh-1,1,2,V,strong_half,400,0.15768703520298005,0.30814920365810394,0.6279279589653016,0.36786327563226223,400,0.42648622393608093,0.7670839726924896,1.3137081503868102,0.834459235817194 +fresh-zh-1,1,2,V,random_half,400,0.0946944683790207,0.20893455296754837,0.416032585501671,0.23785568579100072,400,0.22223201096057893,0.4341427981853485,0.8704446792602539,0.492767509073019 +fresh-zh-1,1,2,V,all_heads,400,0.17003471851348878,0.33705244958400726,0.6679331183433533,0.39753268465399744,400,0.49731341600418094,0.910298764705658,1.5755363106727602,0.987343549206853 +fresh-zh-1,1,2,QKV,weak_half,400,0.043971665576100355,0.07480806857347488,0.12797919362783433,0.08128692565951497,400,0.10788514167070389,0.1659495085477829,0.25618757009506227,0.17556060740724205 +fresh-zh-1,1,2,QKV,strong_half,400,0.2515682548284531,0.39006075263023376,0.57618847489357,0.4058873686566949,400,0.6111020088195801,0.9086828231811523,1.2911702036857606,0.9292143149673939 +fresh-zh-1,1,2,QKV,random_half,400,0.1535023108124733,0.24723856151103973,0.3901005893945694,0.26246471958234907,400,0.3138757228851318,0.5292303264141083,0.7870343148708348,0.5481516802310944 +fresh-zh-1,1,2,QKV,all_heads,400,0.26790088713169097,0.40514326095581055,0.6024578273296357,0.42971948891878126,400,0.7504384517669678,1.0941166281700134,1.5372987389564514,1.104774927198887 +fresh-zh-1,2,3,Q,weak_half,200,0.05399675145745277,0.0818168856203556,0.14206777960062028,0.09009259285405279,200,0.11924902200698853,0.18787774443626404,0.34362818002700807,0.20667793024331332 +fresh-zh-1,2,3,Q,strong_half,200,0.4099469602108002,0.5339741706848145,0.7490169763565063,0.5587046468257904,200,0.7093981266021728,0.9360256791114807,1.3993733167648315,0.99354401409626 +fresh-zh-1,2,3,Q,random_half,200,0.1606532335281372,0.3004520386457443,0.4987666547298431,0.32034398794174196,200,0.3346303790807724,0.5653688907623291,0.8899851322174072,0.5885524506121874 +fresh-zh-1,2,3,Q,all_heads,200,0.43977979421615604,0.5760516822338104,0.7918158829212188,0.5963129077851772,200,0.8348476171493531,1.1361491084098816,1.6960344552993774,1.2002219432592391 +fresh-zh-1,2,3,K,weak_half,200,0.059041570872068405,0.07899310812354088,0.11962006315588945,0.08750347206369043,200,0.13104574382305145,0.1738560050725937,0.2900291085243225,0.19528425343334674 +fresh-zh-1,2,3,K,strong_half,200,0.3587424159049988,0.4287142753601074,0.5362140774726867,0.4395157977938652,200,0.8093313157558442,0.9266857206821442,1.204880714416504,0.9749164399504662 +fresh-zh-1,2,3,K,random_half,200,0.17163306027650835,0.28030669689178467,0.3929991453886032,0.28474022071808575,200,0.3706467866897583,0.548488050699234,0.8294229984283448,0.5737707635760307 +fresh-zh-1,2,3,K,all_heads,200,0.388700345158577,0.4599500745534897,0.5521325409412382,0.4691491121053696,200,0.9481974720954895,1.1239233016967773,1.45943329334259,1.1702006900310515 +fresh-zh-1,2,3,V,weak_half,200,0.0766262635588646,0.11027229577302933,0.17966224253177634,0.12404130116105079,200,0.1749147787690163,0.25942350924015045,0.41556400060653687,0.28096244513988494 +fresh-zh-1,2,3,V,strong_half,200,0.40347065627574924,0.4737415909767151,0.7725899219512938,0.5391813811659812,200,0.9111138582229614,1.0227670669555664,1.7356071949005127,1.18281380712986 +fresh-zh-1,2,3,V,random_half,200,0.18345309048891068,0.34095603227615356,0.5306510269641875,0.35901639781892297,200,0.417214971780777,0.6540262699127197,1.0849312782287597,0.7175682209432125 +fresh-zh-1,2,3,V,all_heads,200,0.417952361702919,0.5285032689571381,0.8517029285430906,0.5842367190122605,200,1.067320454120636,1.2973933219909668,2.2399458646774284,1.4637762427330017 +fresh-zh-1,2,3,QKV,weak_half,200,0.06729187592864036,0.09285303950309753,0.14501096904277802,0.1027342221327126,200,0.1513809636235237,0.2110501229763031,0.3389524519443511,0.23361974608153105 +fresh-zh-1,2,3,QKV,strong_half,200,0.41123413741588594,0.4826723486185074,0.6828670918941497,0.5183154807984829,200,0.859414917230606,0.9670643508434296,1.494808065891266,1.0701916670799256 +fresh-zh-1,2,3,QKV,random_half,200,0.18252907991409303,0.31953395903110504,0.4826954960823059,0.32587722077965736,200,0.3870372474193573,0.6004348397254944,0.9190167844295501,0.6392070814967156 +fresh-zh-1,2,3,QKV,all_heads,200,0.4409913569688797,0.5258795917034149,0.7438874483108521,0.5564920553565025,200,1.0199182391166688,1.1750049591064453,1.9069260478019714,1.3038114207983016 +fresh-zh-1,2,3,JS,weak_half,200,0.0007635956979356706,0.0021704596001654863,0.004974335106089711,0.002622036027460126,200,0.0016013070940971376,0.0036885495064780116,0.008110555168241261,0.00441157413093606 +fresh-zh-1,2,3,JS,strong_half,200,0.03602478355169296,0.07530947029590607,0.13432563245296478,0.08142041601706296,200,0.025066945888102054,0.04762614704668522,0.094714817404747,0.05667232263833284 +fresh-zh-1,2,3,JS,random_half,200,0.023056293465197088,0.04694209061563015,0.08821678385138511,0.052027800804935395,200,0.007390772039070726,0.020091922022402287,0.05617110319435595,0.028588677913649007 +fresh-zh-1,2,3,JS,all_heads,200,0.03738892748951912,0.08661394938826561,0.15178944021463395,0.09058741359971463,200,0.028982629254460334,0.05374104343354702,0.09855992421507832,0.06108389663510025 +fresh-zh-1,2,3,sqrtJS,weak_half,200,0.02763322610408068,0.04658818058669567,0.07052888423204422,0.04794249725528062,200,0.09896892532706261,0.15010160207748413,0.22280997037887573,0.15566823057830334 +fresh-zh-1,2,3,sqrtJS,strong_half,200,0.18980180025100707,0.27442556619644165,0.3665046185255051,0.27545105397701264,200,0.37430735528469083,0.5125177502632141,0.6349055469036102,0.5128714676946402 +fresh-zh-1,2,3,sqrtJS,random_half,200,0.15184295177459717,0.21666111052036285,0.297012647986412,0.22051891624927522,200,0.20104051381349564,0.304886057972908,0.479388952255249,0.3268989910930395 +fresh-zh-1,2,3,sqrtJS,all_heads,200,0.19336212575435638,0.2943022549152374,0.38960163593292235,0.29073272068053485,200,0.5068664371967316,0.663479745388031,0.8529259562492371,0.6685396957397461 +fresh-zh-1,3,4,Q,weak_half,400,0.08877755552530289,0.1906355768442154,0.3309876173734665,0.20427366643212735,400,0.19817024171352388,0.39743661880493164,0.6423090398311616,0.41117362150922415 +fresh-zh-1,3,4,Q,strong_half,400,0.13030964732170106,0.268055722117424,0.7155240237712863,0.3556191158853471,400,0.3494225084781647,0.7949894070625305,2.2821530342102054,1.0170981113612652 +fresh-zh-1,3,4,Q,random_half,400,0.11850653588771821,0.25314533710479736,0.6840757906436921,0.3351429075654596,400,0.2828671753406525,0.6035878360271454,1.3509388208389281,0.7342159500159323 +fresh-zh-1,3,4,Q,all_heads,400,0.1420954406261444,0.306888684630394,0.7893710672855377,0.3947127929143608,400,0.5557760536670685,1.2035991549491882,2.7694889545440677,1.4282717337831854 +fresh-zh-1,3,4,K,weak_half,400,0.12763701826334,0.19483186304569244,0.30266150534152986,0.21024291705340148,400,0.27013990581035613,0.3868144154548645,0.6095517694950104,0.41216555546969175 +fresh-zh-1,3,4,K,strong_half,400,0.17091443389654162,0.26417508721351624,0.46185141503810884,0.2945280092395842,400,0.49152518808841705,0.7354073226451874,1.2997247934341432,0.8189761148393154 +fresh-zh-1,3,4,K,random_half,400,0.1620687633752823,0.2677365243434906,0.4908462405204775,0.3076079587265849,400,0.3643396556377411,0.5715996325016022,0.9841798245906834,0.6291059607639908 +fresh-zh-1,3,4,K,all_heads,400,0.19234354496002198,0.301144003868103,0.5320123553276063,0.3336399665474892,400,0.7813550591468811,1.1287835240364075,1.8791275978088382,1.2311416667699815 +fresh-zh-1,3,4,V,weak_half,400,0.08822526708245278,0.1717580109834671,0.35664437711238867,0.20474327692762018,400,0.17459637075662612,0.3367889076471329,0.6613060474395752,0.38878134854137897 +fresh-zh-1,3,4,V,strong_half,400,0.1068353720009327,0.23760538548231125,0.4541599303483963,0.26727694367989896,400,0.3813276082277298,0.7739077508449554,1.427976202964783,0.8492829108610749 +fresh-zh-1,3,4,V,random_half,400,0.0931014746427536,0.2364826202392578,0.500502735376358,0.2778238266147673,400,0.2730931699275971,0.5634375810623169,1.079581940174103,0.6314834501966834 +fresh-zh-1,3,4,V,all_heads,400,0.11183765903115273,0.25698770582675934,0.4958770334720616,0.28749866392463447,400,0.556857430934906,1.0929275751113892,2.0630701065063475,1.2380642561614514 +fresh-zh-1,3,4,QKV,weak_half,400,0.12944608628749849,0.19706621766090393,0.32124142646789555,0.21473156932741402,400,0.26487993001937865,0.38937266170978546,0.6312589287757874,0.4215213353559375 +fresh-zh-1,3,4,QKV,strong_half,400,0.17727361768484115,0.27701544761657715,0.5620069742202759,0.3265001658722758,400,0.5138821184635163,0.8203077614307404,1.5955090045928957,0.9558513394743204 +fresh-zh-1,3,4,QKV,random_half,400,0.1712821960449219,0.28065013885498047,0.5587266385555267,0.32657942570745946,400,0.36381660103797914,0.6186921298503876,1.126687061786652,0.7058488789200783 +fresh-zh-1,3,4,QKV,all_heads,400,0.1928674966096878,0.30211141705513,0.6164137423038484,0.3600309194251895,400,0.7889724135398865,1.2168751955032349,2.137620258331299,1.377372669875622 +fresh-zh-1,4,5,Q,weak_half,400,0.05576799660921098,0.10025784373283386,0.15577675998210908,0.10335787856020033,400,0.16123872101306916,0.26902781426906586,0.39326150119304665,0.27573458544909957 +fresh-zh-1,4,5,Q,strong_half,400,0.1350233569741249,0.24236910045146942,0.40307160913944245,0.2606338014919311,400,0.3765750110149384,0.6988053619861603,1.0037503242492678,0.69877919010818 +fresh-zh-1,4,5,Q,random_half,400,0.09141657501459122,0.18635692447423935,0.30710877776145934,0.19617343680001795,400,0.26322043836116793,0.48295463621616364,0.740176010131836,0.4923896681331098 +fresh-zh-1,4,5,Q,all_heads,400,0.14631453454494478,0.26939286291599274,0.437625515460968,0.28277991181239487,400,0.5529486596584321,0.9712166786193848,1.3937917470932006,0.974513777717948 +fresh-zh-1,4,5,K,weak_half,400,0.05925469286739827,0.10307718440890312,0.15392619073390962,0.10514573090709746,400,0.16817445605993273,0.26706846058368683,0.3702609807252884,0.2681632469594479 +fresh-zh-1,4,5,K,strong_half,400,0.14917914867401122,0.22241860628128052,0.3328395873308182,0.23246039295569063,400,0.43738883137702944,0.6282206177711487,0.8976162433624268,0.6437146221473813 +fresh-zh-1,4,5,K,random_half,400,0.10047802925109864,0.17267272621393204,0.25870846211910253,0.17790627966634928,400,0.2853825956583023,0.4505874067544937,0.6713049590587616,0.45478436460718513 +fresh-zh-1,4,5,K,all_heads,400,0.16592085212469102,0.23838047683238983,0.3554170072078705,0.24754423982463777,400,0.6115224003791809,0.9048468768596649,1.2517872571945192,0.911877868808806 +fresh-zh-1,4,5,V,weak_half,400,0.030151692405343055,0.07555181160569191,0.1710604429244995,0.08968771325889975,400,0.09685601592063904,0.2000381499528885,0.3791770339012147,0.22596301196143032 +fresh-zh-1,4,5,V,strong_half,400,0.06689317524433136,0.17222751677036285,0.3836905986070634,0.2061547191441059,400,0.2197560101747513,0.4891510307788849,1.084988105297089,0.5999316719733179 +fresh-zh-1,4,5,V,random_half,400,0.05041416510939598,0.13614831864833832,0.3101377248764039,0.16348010992165654,400,0.1574349656701088,0.3465040326118469,0.727318572998047,0.4124986443296075 +fresh-zh-1,4,5,V,all_heads,400,0.06702195554971695,0.18366500735282898,0.40615102946758275,0.22060948628932237,400,0.32456775903701784,0.6919851303100586,1.4499126434326173,0.8258946826681495 +fresh-zh-1,4,5,QKV,weak_half,400,0.06146616935729981,0.10201549157500267,0.152953377366066,0.10406787698157131,400,0.17799552977085115,0.261502206325531,0.384839791059494,0.27152379939332605 +fresh-zh-1,4,5,QKV,strong_half,400,0.15066257268190383,0.22572120279073715,0.35881091058254244,0.2457863415963948,400,0.4328793048858643,0.6357929110527039,1.0055132150650026,0.6917249806970358 +fresh-zh-1,4,5,QKV,random_half,400,0.10997264683246613,0.17862296104431152,0.2879849582910538,0.1895177278481424,400,0.2910709083080292,0.4526078402996063,0.7239212632179263,0.4832642637938261 +fresh-zh-1,4,5,QKV,all_heads,400,0.157893368601799,0.2483942210674286,0.3875706613063813,0.26381413251161573,400,0.6183724403381348,0.8954021334648132,1.3749656796455385,0.9632487814128399 +fresh-zh-1,5,6,Q,weak_half,400,0.08162263929843903,0.1320990025997162,0.20109879970550543,0.13826844365336,400,0.24256522506475447,0.3693459630012512,0.5383547127246857,0.38070347789674996 +fresh-zh-1,5,6,Q,strong_half,400,0.15095633119344712,0.2600850611925125,0.4210688799619677,0.27932014102116226,400,0.4853302538394928,0.7841666042804718,1.0950631499290469,0.7855425517261029 +fresh-zh-1,5,6,Q,random_half,400,0.1196344517171383,0.1950237676501274,0.33364652991294863,0.21302084219641984,400,0.3658181428909302,0.5652031600475311,0.8248552203178408,0.584077067002654 +fresh-zh-1,5,6,Q,all_heads,400,0.16893992125988008,0.2967893034219742,0.4724609524011614,0.3107795799896121,400,0.7430913269519805,1.1477361917495728,1.6089439988136292,1.1662460286170244 +fresh-zh-1,5,6,K,weak_half,400,0.10401069223880768,0.1504753828048706,0.2240424022078515,0.16084511993452907,400,0.2857806742191315,0.39370766282081604,0.5289291739463806,0.40377100452780723 +fresh-zh-1,5,6,K,strong_half,400,0.17918450087308885,0.26103416085243225,0.3833054393529892,0.27496227143332364,400,0.5338803231716156,0.7362784743309021,1.033103358745575,0.7654058936983347 +fresh-zh-1,5,6,K,random_half,400,0.14682336151599884,0.21903035044670105,0.32664437294006354,0.22969845859333873,400,0.3989590346813202,0.5646730959415436,0.8241022765636445,0.5830455023050308 +fresh-zh-1,5,6,K,all_heads,400,0.20395950675010682,0.2988586276769638,0.44880013763904575,0.3116067587770522,400,0.8241071581840516,1.1225058436393738,1.5527066946029664,1.1691768988966942 +fresh-zh-1,5,6,V,weak_half,400,0.06876573711633682,0.13531936705112457,0.2629227310419083,0.15432384436950086,400,0.18456308245658876,0.34114469587802887,0.5228728592395784,0.35366254100576044 +fresh-zh-1,5,6,V,strong_half,400,0.13499078750610352,0.24664560705423355,0.4439507544040681,0.2769244600832462,400,0.4158788651227951,0.6835852563381195,1.1467921614646912,0.747584570273757 +fresh-zh-1,5,6,V,random_half,400,0.09235009551048279,0.19763392210006714,0.37172569036483766,0.2193412135541439,400,0.3016202002763748,0.5010478496551514,0.851252567768097,0.5505481114238501 +fresh-zh-1,5,6,V,all_heads,400,0.13764019012451173,0.27905893325805664,0.5413329005241394,0.3226010990329087,400,0.6337890267372132,0.9958858489990234,1.6384272933006292,1.1012471091002225 +fresh-zh-1,5,6,QKV,weak_half,400,0.10663556009531021,0.149765245616436,0.213547320663929,0.1576207318715751,400,0.29712230861186983,0.39418596029281616,0.5108011960983276,0.39877649378031493 +fresh-zh-1,5,6,QKV,strong_half,400,0.18995745331048966,0.2712521404027939,0.3902842551469803,0.28755593914538624,400,0.6031067907810211,0.7943316996097565,1.03429936170578,0.8138715727627277 +fresh-zh-1,5,6,QKV,random_half,400,0.15813512802124025,0.22214297950267792,0.3162186205387116,0.2305565543845296,400,0.43425395488739016,0.5900538563728333,0.8003608107566834,0.6067851195484399 +fresh-zh-1,5,6,QKV,all_heads,400,0.21847668141126633,0.30972257256507874,0.4561318069696427,0.32754593312740327,400,0.9146241962909699,1.1974874138832092,1.5274685382843018,1.2126480650901794 +fresh-zh-1,6,7,Q,weak_half,200,0.044630038738250735,0.06908540055155754,0.10652023702859878,0.07311033392325043,200,0.11383246928453446,0.1794165074825287,0.27459506392478944,0.18752128813415767 +fresh-zh-1,6,7,Q,strong_half,200,0.25834936201572417,0.31298109889030457,0.39465529918670655,0.323685033172369,200,0.5672750353813172,0.6982413232326508,0.8843709528446198,0.7100403444468975 +fresh-zh-1,6,7,Q,random_half,200,0.11323542147874832,0.22186734527349472,0.29478981792926784,0.21050708495080472,200,0.25739777088165283,0.4542194902896881,0.6096898257732392,0.4420039285719395 +fresh-zh-1,6,7,Q,all_heads,200,0.2778882771730423,0.3303273916244507,0.41822925209999084,0.34459773197770116,200,0.6967215657234191,0.8647677600383759,1.1318518400192261,0.8975616255402565 +fresh-zh-1,6,7,K,weak_half,200,0.05850192904472351,0.07676976919174194,0.10215225890278816,0.07784467065706849,200,0.1374432072043419,0.1957951784133911,0.26449063420295715,0.20038350857794285 +fresh-zh-1,6,7,K,strong_half,200,0.24810760617256167,0.2962874174118042,0.3591116219758986,0.30465917952358723,200,0.6248792290687561,0.7283044159412384,0.9236974537372583,0.753680434525013 +fresh-zh-1,6,7,K,random_half,200,0.12450601756572724,0.21181373298168182,0.30144075155258176,0.21215128745883705,200,0.3020655155181885,0.4590883404016495,0.6441370487213135,0.468495799601078 +fresh-zh-1,6,7,K,all_heads,200,0.2661030113697052,0.3140828162431717,0.38857126533985126,0.3246633425354958,200,0.7858375787734987,0.9240201115608215,1.1293065905570978,0.9540639388561248 +fresh-zh-1,6,7,V,weak_half,200,0.07523173317313195,0.09802470356225967,0.13378061950206757,0.10092367269098759,200,0.1804918020963669,0.25908592343330383,0.3507958650588989,0.26191206589341165 +fresh-zh-1,6,7,V,strong_half,200,0.3202996850013733,0.36071527004241943,0.44959115982055664,0.37323995888233186,200,0.7397605359554291,0.8810235261917114,1.1074213147163388,0.8972787618637085 +fresh-zh-1,6,7,V,random_half,200,0.1626923441886902,0.2626984268426895,0.3519802153110504,0.2571912503242493,200,0.3875861883163452,0.573321133852005,0.756132024526596,0.5709240680932999 +fresh-zh-1,6,7,V,all_heads,200,0.33642291128635404,0.38896091282367706,0.4716356217861175,0.3977210849523544,200,0.9605062544345856,1.1428414583206177,1.3685777783393853,1.1591908264160156 +fresh-zh-1,6,7,QKV,weak_half,200,0.06301232427358627,0.08426452055573463,0.11138175055384636,0.08555747766047717,200,0.15677377879619597,0.21932102739810944,0.28658311665058134,0.22165599457919596 +fresh-zh-1,6,7,QKV,strong_half,200,0.2899006187915802,0.32590973377227783,0.4082955956459045,0.336367521584034,200,0.686147129535675,0.7741285562515259,0.9817734837532043,0.7996673840284347 +fresh-zh-1,6,7,QKV,random_half,200,0.14530814737081527,0.23306860774755478,0.31072844564914703,0.22889333486557006,200,0.33507010340690613,0.4985084980726242,0.6602601289749146,0.5024830457568169 +fresh-zh-1,6,7,QKV,all_heads,200,0.30483112931251527,0.3500949740409851,0.42342236936092376,0.35841443166136744,200,0.8571053206920624,1.0059685111045837,1.183649742603302,1.021323380470276 +fresh-zh-1,6,7,JS,weak_half,200,0.000492150749778375,0.0015077641000971198,0.0045888546854257585,0.002074052241659956,200,0.0011745220515877008,0.0028378808638080955,0.009065340273082256,0.004262433346520993 +fresh-zh-1,6,7,JS,strong_half,200,0.02876556143164635,0.0638500452041626,0.11921769604086875,0.07131386235356331,200,0.019655799120664598,0.03923086263239384,0.06797220408916473,0.042609280860051515 +fresh-zh-1,6,7,JS,random_half,200,0.011326245684176684,0.031219650991261005,0.06300913020968436,0.035424025055253876,200,0.00624479907564819,0.01716148667037487,0.04215944074094295,0.021583106761681847 +fresh-zh-1,6,7,JS,all_heads,200,0.03103185072541237,0.07097619399428368,0.12892837077379227,0.07686727553606033,200,0.0218641459941864,0.04420311748981476,0.07254369854927062,0.04687171423342079 +fresh-zh-1,6,7,sqrtJS,weak_half,200,0.02218447029590607,0.03882993571460247,0.06774105578660965,0.04190785993356258,200,0.0837737001478672,0.13177590817213058,0.2278826281428337,0.1442319119349122 +fresh-zh-1,6,7,sqrtJS,strong_half,200,0.16959669888019563,0.25268396735191345,0.3452784091234207,0.2580921309441328,200,0.3297521352767944,0.458116739988327,0.6000279068946839,0.4632533569633961 +fresh-zh-1,6,7,sqrtJS,random_half,200,0.10642305016517639,0.17669080197811127,0.2510161340236664,0.17931865451857448,200,0.18127433806657792,0.29021403193473816,0.43702368438243866,0.29812911942601206 +fresh-zh-1,6,7,sqrtJS,all_heads,200,0.17615844756364823,0.26641298830509186,0.359064581990242,0.26766244806349276,200,0.43271684646606445,0.6038041114807129,0.7724204778671264,0.6074852642416954 +fresh-zh-1,7,8,Q,weak_half,400,0.12256367206573486,0.20214895904064178,0.4163078874349595,0.24106176905333995,400,0.2904431581497192,0.4710287004709244,0.8929737389087677,0.5511688451841473 +fresh-zh-1,7,8,Q,strong_half,400,0.18436692953109743,0.2792464345693588,0.5555767416954042,0.33521180389449,400,0.6539463698863984,1.0762452483177185,1.9961119890213017,1.2206342507898809 +fresh-zh-1,7,8,Q,random_half,400,0.17398671060800552,0.30122195184230804,0.6057721614837648,0.3475493448227644,400,0.43449713587760924,0.7587410807609558,1.5415699481964114,0.8872211270034314 +fresh-zh-1,7,8,Q,all_heads,400,0.19847133308649062,0.2861030697822571,0.4456568717956544,0.31451957946643233,400,0.9590097606182099,1.5364474058151245,2.8689801931381225,1.771803096830845 +fresh-zh-1,7,8,K,weak_half,400,0.046465735509991654,0.18298520147800446,0.35136889815330513,0.19814407165395095,400,0.12310988083481796,0.4113724082708359,0.6867121219635011,0.4413666472397745 +fresh-zh-1,7,8,K,strong_half,400,0.05793619826436044,0.23408634215593338,0.39703599512577065,0.24904022382572294,400,0.1893985688686371,0.9129117429256439,1.476654601097107,0.9322665578313172 +fresh-zh-1,7,8,K,random_half,400,0.09776061549782757,0.2976543754339218,0.5133859217166901,0.30850857332581655,400,0.15572057366371161,0.6666693389415741,1.0792351722717286,0.6907902594469487 +fresh-zh-1,7,8,K,all_heads,400,0.06462859995663167,0.23689372092485428,0.3593016654253006,0.2377785169845447,400,0.2979481488466263,1.3236027359962463,2.1348606586456302,1.3736332034133374 +fresh-zh-1,7,8,V,weak_half,400,0.07659564018249512,0.16163919866085052,0.334882864356041,0.18833234277553856,400,0.17523235082626343,0.34737639129161835,0.6507042944431305,0.3891457209922373 +fresh-zh-1,7,8,V,strong_half,400,0.09209293052554131,0.22275249660015106,0.46667803227901467,0.2607644669339061,400,0.3897214263677597,0.9488252103328705,1.6349377274513246,0.9760747821629048 +fresh-zh-1,7,8,V,random_half,400,0.11343886852264405,0.2552698850631714,0.5673084199428559,0.30713419082574545,400,0.26434861719608305,0.6148198544979095,1.2529494762420657,0.6877283260971307 +fresh-zh-1,7,8,V,all_heads,400,0.08976964578032494,0.20600531250238419,0.3951597183942795,0.22936863113194705,400,0.5749102354049683,1.344321370124817,2.2224256515502945,1.3652205036580563 +fresh-zh-1,7,8,QKV,weak_half,400,0.11926292851567269,0.1958867534995079,0.3691331326961518,0.21952526157721877,400,0.2738349914550781,0.43825390934944153,0.7264724135398865,0.48786259043961766 +fresh-zh-1,7,8,QKV,strong_half,400,0.16446534395217896,0.2626783400774002,0.4516425997018814,0.2986487899534404,400,0.6440905332565308,1.067484200000763,1.5564946055412292,1.1139849627763032 +fresh-zh-1,7,8,QKV,random_half,400,0.18954066336154937,0.31212614476680756,0.5175672948360444,0.3415677944198251,400,0.4273326396942139,0.7380891442298889,1.2356342792510988,0.8049949992075562 +fresh-zh-1,7,8,QKV,all_heads,400,0.16933170408010484,0.2573424130678177,0.37787349820137034,0.2745602697879076,400,0.927498334646225,1.5033814907073975,2.265283608436585,1.6018475522100926 +fresh-zh-1,8,9,Q,weak_half,400,0.07247661501169204,0.11240289360284805,0.16622977405786515,0.11596273303963243,400,0.20690689831972123,0.31034450232982635,0.44291880130767824,0.3192695660144091 +fresh-zh-1,8,9,Q,strong_half,400,0.14283609092235566,0.2269045189023018,0.3336938589811325,0.23684374550357462,400,0.4433826267719269,0.659704715013504,0.8977041840553284,0.6698895494639874 +fresh-zh-1,8,9,Q,random_half,400,0.11141371726989746,0.1724121794104576,0.2621489852666855,0.18310893481597304,400,0.31521634459495546,0.48340854048728943,0.6795005023479462,0.4966938193514943 +fresh-zh-1,8,9,Q,all_heads,400,0.16711665987968444,0.2534511089324951,0.36788355410099044,0.26194717628881337,400,0.6906953930854798,0.9759601950645447,1.3041144490242005,0.9891591170430183 +fresh-zh-1,8,9,K,weak_half,400,0.07246405482292176,0.11668133735656738,0.175839638710022,0.12250027022324503,400,0.19773499965667726,0.2904649078845978,0.44829008281230925,0.30719231214374304 +fresh-zh-1,8,9,K,strong_half,400,0.14385293424129486,0.20046809315681458,0.2919215381145478,0.21064413478598,400,0.4581241965293884,0.611322283744812,0.8378353893756867,0.6368358009308577 +fresh-zh-1,8,9,K,random_half,400,0.12043995857238769,0.1730336770415306,0.26333478689193734,0.18073420161381365,400,0.3182948440313339,0.4627998322248459,0.6461710393428802,0.4729698371514678 +fresh-zh-1,8,9,K,all_heads,400,0.1629441395401955,0.23235876858234406,0.3232919633388519,0.2380711136572063,400,0.68043452501297,0.9005166888237,1.261199188232422,0.9440281166136265 +fresh-zh-1,8,9,V,weak_half,400,0.05026836693286896,0.1030503548681736,0.1926133528351784,0.11547209623269737,400,0.14484444111585618,0.27241581678390503,0.43689213395118714,0.2877457766234875 +fresh-zh-1,8,9,V,strong_half,400,0.09586298242211343,0.1930147409439087,0.32488626837730405,0.20504348921589555,400,0.2885640889406204,0.5663753151893616,0.9333638370037081,0.6018495421111584 +fresh-zh-1,8,9,V,random_half,400,0.07789478674530984,0.14984657615423203,0.2591643929481507,0.1632109899353236,400,0.21465063989162445,0.4107406884431839,0.6974971473217011,0.444117740765214 +fresh-zh-1,8,9,V,all_heads,400,0.11281683370471002,0.21647904813289642,0.3695981085300446,0.23007617874071,400,0.4647453725337983,0.8514449596405029,1.372711777687073,0.8895953219383955 +fresh-zh-1,8,9,QKV,weak_half,400,0.07817873656749726,0.11752442270517349,0.1664638236165047,0.12188695802353322,400,0.2142637014389038,0.3133063018321991,0.4293359726667405,0.31876118324697017 +fresh-zh-1,8,9,QKV,strong_half,400,0.1591764658689499,0.22143830358982086,0.3013652116060257,0.22594965802505612,400,0.5066094100475311,0.6682660579681396,0.8492630898952485,0.6732346745580435 +fresh-zh-1,8,9,QKV,random_half,400,0.12605225145816804,0.1788506656885147,0.23584195077419282,0.18244648838415742,400,0.3481952279806137,0.4929962158203125,0.6406627237796784,0.49725816696882247 +fresh-zh-1,8,9,QKV,all_heads,400,0.18547458350658416,0.24602121114730835,0.3285394847393036,0.2521421314403415,400,0.7347855746746064,0.9900411367416382,1.2494902253150941,0.9919958572089672 +fresh-zh-1,9,10,Q,weak_half,400,0.06773708760738373,0.11841709911823273,0.18038420230150223,0.12180521365255118,400,0.17921194136142732,0.3008343279361725,0.4628704071044923,0.3168759752251208 +fresh-zh-1,9,10,Q,strong_half,400,0.13857623636722566,0.24519570171833038,0.40940831303596503,0.26388716923072936,400,0.41984589099884034,0.7198978960514069,1.0580605506896974,0.7415574808418751 +fresh-zh-1,9,10,Q,random_half,400,0.10521642565727235,0.18340732902288437,0.30094167888164525,0.19506874246522785,400,0.29067459106445315,0.49722522497177124,0.7818213224411011,0.5253341678157448 +fresh-zh-1,9,10,Q,all_heads,400,0.15452902466058732,0.2760113775730133,0.45418669879436496,0.29486150892451407,400,0.6123335957527161,1.0435316562652588,1.5125911712646487,1.0584334572404623 +fresh-zh-1,9,10,K,weak_half,400,0.07287878692150117,0.1162058487534523,0.17366309463977814,0.12091243754141033,400,0.18735334277153015,0.28096771240234375,0.3908685892820358,0.2878991183638573 +fresh-zh-1,9,10,K,strong_half,400,0.14225213080644608,0.21243591606616974,0.31798889935016633,0.22529558187350632,400,0.4308472841978073,0.6051242053508759,0.8061724781990052,0.6164266370981931 +fresh-zh-1,9,10,K,random_half,400,0.11605458781123162,0.18240205943584442,0.2697263985872269,0.18750731946900487,400,0.2942979007959366,0.43626560270786285,0.6254877567291259,0.45323856856673955 +fresh-zh-1,9,10,K,all_heads,400,0.15947582423686982,0.2458697408437729,0.34799642264842995,0.25606418328359726,400,0.6221934735774994,0.8891855180263519,1.1792654395103457,0.9043257541954517 +fresh-zh-1,9,10,V,weak_half,400,0.058301230892539024,0.09646684303879738,0.1715340942144394,0.1089102391153574,400,0.14986272901296616,0.23593128472566605,0.3977706104516983,0.261308101080358 +fresh-zh-1,9,10,V,strong_half,400,0.09634882733225823,0.17709998786449432,0.3287752866744995,0.19845054027624429,400,0.3170677751302719,0.4882224351167679,0.8094505548477174,0.535103571973741 +fresh-zh-1,9,10,V,random_half,400,0.07994768247008324,0.13912361860275269,0.264794996380806,0.1596498021297157,400,0.22556192576885226,0.3619781583547592,0.6200000405311586,0.3978969208151102 +fresh-zh-1,9,10,V,all_heads,400,0.11607617363333703,0.21699212491512299,0.3696089446544648,0.23190834769979118,400,0.4736921221017838,0.7299782633781433,1.210689949989319,0.7964116725325584 +fresh-zh-1,9,10,QKV,weak_half,400,0.07601049840450287,0.11840922012925148,0.16417539268732073,0.12110782583244145,400,0.20238763391971587,0.2979876697063446,0.40853255391120913,0.3021149646490812 +fresh-zh-1,9,10,QKV,strong_half,400,0.1597129851579666,0.2344036102294922,0.32072209715843203,0.23997954534366728,400,0.44837206304073335,0.6696524918079376,0.8873418092727661,0.6743833009898662 +fresh-zh-1,9,10,QKV,random_half,400,0.12379511669278145,0.18230856955051422,0.25906082391738905,0.18873877223581076,400,0.32346543073654177,0.47984737157821655,0.6752042651176453,0.48723199546337126 +fresh-zh-1,9,10,QKV,all_heads,400,0.17665915191173553,0.26725006103515625,0.3686117142438889,0.27202690720558165,400,0.654531466960907,0.9747728407382965,1.2900494813919068,0.9764982675760985 +fresh-zh-1,10,11,Q,weak_half,200,0.07315033599734307,0.10080759599804878,0.13348732739686966,0.1036814227886498,200,0.19117356538772584,0.2713441550731659,0.3522419542074203,0.2711084973067045 +fresh-zh-1,10,11,Q,strong_half,200,0.17135028094053267,0.22562847286462784,0.2819775611162185,0.22858978793025017,200,0.43651853799819945,0.6072356998920441,0.745236611366272,0.6011908978223801 +fresh-zh-1,10,11,Q,random_half,200,0.11988586187362671,0.16981659829616547,0.22604970335960387,0.17211561594158412,200,0.3002244144678116,0.4238482713699341,0.5635224938392639,0.4299582788348198 +fresh-zh-1,10,11,Q,all_heads,200,0.196297487616539,0.25928835570812225,0.32484194040298464,0.2593411233276129,200,0.6438421249389649,0.872416615486145,1.0983223557472228,0.8722993984818459 +fresh-zh-1,10,11,K,weak_half,200,0.05984708443284035,0.08421061187982559,0.10727114900946617,0.08400685587897896,200,0.15494146049022675,0.22098806500434875,0.27376148104667664,0.2173728848993778 +fresh-zh-1,10,11,K,strong_half,200,0.21974741220474245,0.2842578887939453,0.3453286290168762,0.28566086523234846,200,0.5280612409114838,0.6487925052642822,0.8054875791072845,0.6574551117420196 +fresh-zh-1,10,11,K,random_half,200,0.1108615331351757,0.1936115100979805,0.2872999906539917,0.19759830348193647,200,0.28392315506935123,0.4269130825996399,0.5600059211254119,0.4302507738023996 +fresh-zh-1,10,11,K,all_heads,200,0.24518719464540484,0.3005325347185135,0.3648728281259536,0.30628038257360457,200,0.7055036425590515,0.8617449402809143,0.9929532289505003,0.8748280000686646 +fresh-zh-1,10,11,V,weak_half,200,0.08327115029096603,0.12298920005559921,0.14664961397647858,0.11682839337736368,200,0.22837378382682802,0.32463522255420685,0.3833968520164489,0.312385072261095 +fresh-zh-1,10,11,V,strong_half,200,0.19371578097343445,0.22748331725597382,0.2900074988603591,0.24077702559530734,200,0.5435559391975403,0.6613179743289948,0.8311548709869384,0.6737473028898239 +fresh-zh-1,10,11,V,random_half,200,0.14260331094264983,0.18101278692483902,0.2388536438345909,0.18710535854101182,200,0.3593408018350601,0.47883372008800507,0.6069994986057281,0.488108162432909 +fresh-zh-1,10,11,V,all_heads,200,0.21605052947998046,0.2575386017560959,0.32805477380752557,0.26688400745391844,200,0.7727445721626282,0.975719690322876,1.1745961308479307,0.9861323750019073 +fresh-zh-1,10,11,QKV,weak_half,200,0.07640505433082581,0.10442130640149117,0.128546079993248,0.10277458667755127,200,0.20433848798274995,0.277020663022995,0.33793311715126034,0.27233862176537516 +fresh-zh-1,10,11,QKV,strong_half,200,0.208316870033741,0.2544317841529846,0.2958696514368057,0.2547766848653555,200,0.5469081044197083,0.659739762544632,0.7887015700340271,0.6658182474970817 +fresh-zh-1,10,11,QKV,random_half,200,0.13420266062021255,0.18674493581056595,0.24521257579326627,0.1884442066773772,200,0.3310825139284134,0.45819905400276184,0.5765936851501465,0.4630968165397644 +fresh-zh-1,10,11,QKV,all_heads,200,0.2284634992480278,0.2809319794178009,0.3233322948217392,0.2800913665443659,200,0.7646997034549713,0.9415693879127502,1.1005946516990661,0.9381568691134453 +fresh-zh-1,10,11,JS,weak_half,200,0.0009009615168906748,0.0031164383981376886,0.00891707073897123,0.004110419770076987,200,0.0015200424473732711,0.003671261598356068,0.007784343510866164,0.004227555798424873 +fresh-zh-1,10,11,JS,strong_half,200,0.016405019909143448,0.03043436072766781,0.07099740728735923,0.03859887976432219,200,0.01339836409315467,0.023128618486225605,0.05012134201824665,0.029376834339927882 +fresh-zh-1,10,11,JS,random_half,200,0.006406870065256954,0.0166408009827137,0.04111998602747917,0.021699520393740387,200,0.004981679515913129,0.012044424656778574,0.028448386117815968,0.016769534255145116 +fresh-zh-1,10,11,JS,all_heads,200,0.017791744880378248,0.03878171928226948,0.07899344190955161,0.04523044921224937,200,0.016045292653143407,0.028061560355126858,0.05576590038836002,0.03360439012292773 +fresh-zh-1,10,11,sqrtJS,weak_half,200,0.030016020499169828,0.05582505464553833,0.0944271445274353,0.0587089966237545,200,0.10110943093895912,0.16046104580163956,0.23489216566085813,0.16502912521362303 +fresh-zh-1,10,11,sqrtJS,strong_half,200,0.1280820697546005,0.17445337772369385,0.2664526730775833,0.18731017913669348,200,0.28618484139442446,0.38825634121894836,0.5202878832817077,0.4015352885425091 +fresh-zh-1,10,11,sqrtJS,random_half,200,0.08004223108291626,0.12899906188249588,0.20278004854917525,0.13703191483393312,200,0.1772498890757561,0.26817555725574493,0.4014991492033005,0.2794022048637271 +fresh-zh-1,10,11,sqrtJS,all_heads,200,0.1333855628967285,0.19692932069301605,0.28105699717998506,0.2028772597014904,200,0.40895286202430725,0.553448349237442,0.7309623777866363,0.5665644085407258 +fresh-zh-1,11,12,Q,weak_half,400,0.10215479657053947,0.1657293662428856,0.3357654303312302,0.19619444590993226,400,0.24090565145015716,0.3410186171531677,0.5038730084896088,0.3617283037304878 +fresh-zh-1,11,12,Q,strong_half,400,0.13013015687465668,0.20637840777635574,0.35754604637622833,0.22453830210492015,400,0.4639055997133255,0.7209613919258118,1.1560844779014587,0.7719487391412259 +fresh-zh-1,11,12,Q,random_half,400,0.12381389066576959,0.21136575937271118,0.3912459820508958,0.23530046581290662,400,0.31300667226314544,0.5333824753761292,0.862956702709198,0.5658085729554295 +fresh-zh-1,11,12,Q,all_heads,400,0.12331643551588059,0.17818573117256165,0.25711416602134707,0.18548751808702946,400,0.711259400844574,1.0567578673362732,1.6549084544181825,1.13367704115808 +fresh-zh-1,11,12,K,weak_half,400,0.08084328770637512,0.1454273834824562,0.27749171257019045,0.16616901079192758,400,0.17427084147930144,0.2685410976409912,0.4484488844871521,0.2895404239185154 +fresh-zh-1,11,12,K,strong_half,400,0.09674982652068138,0.1561223492026329,0.25686334371566777,0.17056298185139895,400,0.3052971690893173,0.5290720164775848,0.9288893699645998,0.5772540457174182 +fresh-zh-1,11,12,K,random_half,400,0.10392512977123261,0.18607641011476517,0.36494546234607705,0.22218815599568187,400,0.21842414289712905,0.39015501737594604,0.7097547948360443,0.4351468885503709 +fresh-zh-1,11,12,K,all_heads,400,0.10123785883188248,0.1537189483642578,0.2425813078880311,0.1675046616792679,400,0.4825268000364304,0.7918677926063538,1.3957195162773133,0.8667944663763046 +fresh-zh-1,11,12,V,weak_half,400,0.0621289111673832,0.12436965852975845,0.23850261270999915,0.14018684964161365,400,0.13552124798297882,0.22920261323451996,0.42135479748249066,0.25454884245991705 +fresh-zh-1,11,12,V,strong_half,400,0.08001176938414574,0.13327914476394653,0.26810979247093203,0.15498670453671365,400,0.24896615296602248,0.44784627854824066,0.9453880190849305,0.5235174075886607 +fresh-zh-1,11,12,V,random_half,400,0.07114746198058129,0.14786117523908615,0.3270563364028932,0.1860357138235122,400,0.18053750693798065,0.32924653589725494,0.6984821856021886,0.38977345019578935 +fresh-zh-1,11,12,V,all_heads,400,0.078624626994133,0.13846991956233978,0.24511605203151704,0.15220589406788348,400,0.3828774690628052,0.6756212711334229,1.3400470137596132,0.7780662517622113 +fresh-zh-1,11,12,QKV,weak_half,400,0.09919282719492913,0.16120554506778717,0.28363404273986814,0.17588178424164652,400,0.21480963230133057,0.30338504910469055,0.4459644198417664,0.31649471338838336 +fresh-zh-1,11,12,QKV,strong_half,400,0.12401885092258454,0.18434961140155792,0.2785502105951309,0.19250067112967373,400,0.40059158504009246,0.6264435350894928,0.9444568812847137,0.662094437405467 +fresh-zh-1,11,12,QKV,random_half,400,0.12398836314678192,0.20562929660081863,0.3382799983024598,0.22640024630352854,400,0.2842820167541504,0.457336962223053,0.7411015450954437,0.4894951446354389 +fresh-zh-1,11,12,QKV,all_heads,400,0.12292529940605164,0.16693364083766937,0.23755992203950882,0.1746005929261446,400,0.6279415905475616,0.9242853224277496,1.3742050528526306,0.9785891487449407 +fresh-zh-1,12,13,Q,weak_half,400,0.042724504694342615,0.06851844862103462,0.10168379694223405,0.07029165489599108,400,0.12525956630706786,0.18926294147968292,0.27061432898044585,0.1959054318256676 +fresh-zh-1,12,13,Q,strong_half,400,0.09328480735421181,0.1472376585006714,0.2615198284387589,0.16504959688521922,400,0.2922026991844177,0.439471110701561,0.6846163988113404,0.4643231889232993 +fresh-zh-1,12,13,Q,random_half,400,0.06756708696484566,0.11048552021384239,0.19476523399353027,0.12202541222795844,400,0.1925196960568428,0.31183427572250366,0.47861541211605074,0.33053834073245525 +fresh-zh-1,12,13,Q,all_heads,400,0.10159314647316933,0.15777436643838882,0.2723366618156433,0.17519856724888086,400,0.4287725806236267,0.6291517913341522,0.9183653235435489,0.6602286210656166 +fresh-zh-1,12,13,K,weak_half,400,0.040515297278761866,0.06703095510601997,0.10065712183713918,0.07086932587902993,400,0.11352628320455552,0.17797140777111053,0.2578406363725662,0.1821184110082686 +fresh-zh-1,12,13,K,strong_half,400,0.08639253452420234,0.1276923418045044,0.19838304519653321,0.13538813623599708,400,0.25438788831233977,0.36494405567646027,0.5332014262676239,0.3839222073927522 +fresh-zh-1,12,13,K,random_half,400,0.06938333958387376,0.10581229254603386,0.16611133813858037,0.1126271313149482,400,0.1751003921031952,0.2726609706878662,0.4084711104631424,0.2827327658981085 +fresh-zh-1,12,13,K,all_heads,400,0.09197790399193764,0.14057958871126175,0.21102984547615053,0.14751550318673254,400,0.3796079456806183,0.5419529676437378,0.7618788957595826,0.5660406196117401 +fresh-zh-1,12,13,V,weak_half,400,0.03465876765549183,0.056747930124402046,0.09487150609493256,0.06190093896817416,400,0.0992879755795002,0.1453903466463089,0.21946086138486864,0.15548876747488977 +fresh-zh-1,12,13,V,strong_half,400,0.06611917465925217,0.10798169672489166,0.17892002016305925,0.11678785809781403,400,0.21644230037927628,0.326924204826355,0.4825355976819992,0.3413913020864129 +fresh-zh-1,12,13,V,random_half,400,0.05473507679998875,0.08902623876929283,0.15109524875879293,0.09869084949139506,400,0.16033811420202257,0.2333817034959793,0.3567052453756333,0.24836937338113785 +fresh-zh-1,12,13,V,all_heads,400,0.07274657338857651,0.11921241879463196,0.1910336270928383,0.12790479290299117,400,0.338055619597435,0.4774762690067291,0.6925039589405061,0.49688007023185493 +fresh-zh-1,12,13,QKV,weak_half,400,0.04939929880201817,0.06814679130911827,0.09747000187635424,0.06994277911726385,400,0.13406059592962266,0.1827833503484726,0.24772240370512014,0.18534162597730755 +fresh-zh-1,12,13,QKV,strong_half,400,0.10284400284290314,0.140559583902359,0.19667428582906726,0.14644663983024656,400,0.31987280249595645,0.4114132970571518,0.5506049692630769,0.4229177590459585 +fresh-zh-1,12,13,QKV,random_half,400,0.07788905948400497,0.11318828165531158,0.15897328704595567,0.11636936500668525,400,0.21026905477046967,0.2960865795612335,0.40589936077594757,0.30409911964088676 +fresh-zh-1,12,13,QKV,all_heads,400,0.11306013390421868,0.15314044058322906,0.21001255512237552,0.15760981559753418,400,0.46751987338066103,0.6001860499382019,0.7737954795360565,0.6082593883574009 +fresh-zh-1,13,14,Q,weak_half,400,0.031631321832537654,0.05625314451754093,0.10134881511330605,0.06258326167939231,400,0.09200013503432274,0.15353454649448395,0.25263353884220124,0.16492452405393124 +fresh-zh-1,13,14,Q,strong_half,400,0.0879149079322815,0.16087543964385986,0.2630733221769333,0.16836948637850582,400,0.24690059572458267,0.4471454620361328,0.6865552604198456,0.4544835183210671 +fresh-zh-1,13,14,Q,random_half,400,0.06001562960445882,0.11385346949100494,0.1887395426630974,0.1210738734761253,400,0.16211334168910982,0.29622484743595123,0.4765252083539963,0.30995401899330316 +fresh-zh-1,13,14,Q,all_heads,400,0.09675953090190888,0.1727834865450859,0.2841507881879807,0.18334773622918873,400,0.35818194150924687,0.6057678461074829,0.9266265809535983,0.6194080414809287 +fresh-zh-1,13,14,K,weak_half,400,0.036471526697278024,0.06618835777044296,0.11796363443136215,0.07219005833147094,400,0.09184312149882318,0.1627291738986969,0.26512554287910467,0.17117465760558845 +fresh-zh-1,13,14,K,strong_half,400,0.10304497852921486,0.16782651841640472,0.2900669902563095,0.18315087881870568,400,0.2687643587589264,0.4580760598182678,0.696693480014801,0.4737547692656517 +fresh-zh-1,13,14,K,random_half,400,0.07158857733011247,0.12994074821472168,0.23321569710969928,0.14334922715090215,400,0.1637290060520172,0.308560848236084,0.5122513353824617,0.3216987919062376 +fresh-zh-1,13,14,K,all_heads,400,0.10938302800059319,0.1803639903664589,0.296254912018776,0.19324435868300496,400,0.3721213042736054,0.6305268406867981,0.9718979418277741,0.6449294289946557 +fresh-zh-1,13,14,V,weak_half,400,0.02666907943785191,0.05007386952638626,0.11246339157223702,0.061448822694364934,400,0.07445840165019035,0.12291181832551956,0.2558824628591538,0.14849124061875046 +fresh-zh-1,13,14,V,strong_half,400,0.07395830824971199,0.1330777406692505,0.2874425321817399,0.16520903975702822,400,0.21644818335771562,0.35751597583293915,0.6462142109870911,0.4121435910835862 +fresh-zh-1,13,14,V,random_half,400,0.05592651925981045,0.09383120387792587,0.21180316209793099,0.12041821557097138,400,0.13614681661129,0.24350009858608246,0.48390524387359624,0.28002955509349703 +fresh-zh-1,13,14,V,all_heads,400,0.08251244202256204,0.1475631296634674,0.295827516913414,0.17515847254544498,400,0.2995422065258026,0.48480285704135895,0.9077605545520783,0.5606348321214318 +fresh-zh-1,13,14,QKV,weak_half,400,0.03693911582231522,0.061812980100512505,0.10715146884322166,0.06830849323421717,400,0.09671626389026641,0.1571682244539261,0.25547077655792244,0.16980357254855336 +fresh-zh-1,13,14,QKV,strong_half,400,0.11010506302118302,0.16651400178670883,0.2686866074800492,0.1808051243238151,400,0.3140480935573578,0.45976419746875763,0.6702876448631288,0.4763926822319627 +fresh-zh-1,13,14,QKV,random_half,400,0.07771690562367439,0.12265603244304657,0.21154495477676402,0.13481565448455513,400,0.18774264454841613,0.3032141774892807,0.48368785381317136,0.3228048492595553 +fresh-zh-1,13,14,QKV,all_heads,400,0.12191290035843849,0.1795101910829544,0.2769842267036438,0.19246285488829018,400,0.4210603922605515,0.6132193505764008,0.9099863886833195,0.6461962567269802 +fresh-zh-1,14,15,Q,weak_half,200,0.0631001576781273,0.09207073226571083,0.12801062315702438,0.09526140287518502,200,0.15712271332740785,0.24157647043466568,0.336358779668808,0.2477959617599845 +fresh-zh-1,14,15,Q,strong_half,200,0.20422755926847458,0.2697185128927231,0.3412438690662384,0.2738436217606068,200,0.49408584237098696,0.6688079833984375,0.8648501455783844,0.6703512765467167 +fresh-zh-1,14,15,Q,random_half,200,0.13073387295007705,0.1965862736105919,0.27093083858489986,0.19893720906227827,200,0.3176268845796585,0.4653925597667694,0.6390501439571381,0.4703691750764847 +fresh-zh-1,14,15,Q,all_heads,200,0.2241092160344124,0.2812371104955673,0.3516690254211426,0.2858240742981434,200,0.6577422261238098,0.9171648323535919,1.1852634906768797,0.9181472364068032 +fresh-zh-1,14,15,K,weak_half,200,0.03918342478573322,0.049769796431064606,0.06421543657779694,0.051490013180300595,200,0.0936107188463211,0.12203481420874596,0.15432564318180084,0.1257441356033087 +fresh-zh-1,14,15,K,strong_half,200,0.17188602536916733,0.22778571397066116,0.3678930699825287,0.25282074712216857,200,0.3598978310823441,0.5395549535751343,0.7789157629013062,0.5587836657464504 +fresh-zh-1,14,15,K,random_half,200,0.08957784697413444,0.16862497478723526,0.30518664717674254,0.1771112827397883,200,0.20003172159194946,0.33938515186309814,0.5725238978862762,0.35973493821918967 +fresh-zh-1,14,15,K,all_heads,200,0.181875778734684,0.23612432926893234,0.3708422183990478,0.26158237010240554,200,0.45582233965396884,0.6611933410167694,0.9285608649253845,0.6845277935266495 +fresh-zh-1,14,15,V,weak_half,200,0.03787815235555172,0.053609058260917664,0.07619359120726585,0.056342262448742984,200,0.09143951535224915,0.12571856379508972,0.17327781021595,0.13260096549987793 +fresh-zh-1,14,15,V,strong_half,200,0.12375563383102417,0.1901002898812294,0.3196308642625808,0.20451857842504978,200,0.29888027906417847,0.43949030339717865,0.6810161113739014,0.4584863719344139 +fresh-zh-1,14,15,V,random_half,200,0.0846713624894619,0.13521598279476166,0.2646415442228317,0.15138852503150702,200,0.18196625113487244,0.27942128479480743,0.49971449971199033,0.3128007492423058 +fresh-zh-1,14,15,V,all_heads,200,0.13287830501794815,0.20351067185401917,0.3312030762434005,0.21785607635974885,200,0.39862508177757267,0.5538661181926727,0.8149181783199307,0.5910873407125473 +fresh-zh-1,14,15,QKV,weak_half,200,0.05167605951428413,0.06841527670621872,0.09357256814837454,0.07120389617979526,200,0.1276782512664795,0.1784556433558464,0.23929117172956466,0.1826014292240143 +fresh-zh-1,14,15,QKV,strong_half,200,0.1996445909142494,0.25597330927848816,0.29727073907852175,0.25209633380174634,200,0.4954728066921234,0.6071159839630127,0.7396595478057861,0.6154910288751125 +fresh-zh-1,14,15,QKV,random_half,200,0.12891489416360855,0.17533809691667557,0.24822963923215866,0.18365731401368976,200,0.28962488770484923,0.40496107935905457,0.5471580922603607,0.4166713547706604 +fresh-zh-1,14,15,QKV,all_heads,200,0.21269166320562363,0.2670910805463791,0.3055910676717758,0.2626703421771526,200,0.6458287894725799,0.7748749256134033,0.9652682244777679,0.7980924570560455 +fresh-zh-1,14,15,JS,weak_half,200,0.0010359299136325716,0.0025107450783252716,0.007744135055691001,0.003716461525327759,200,0.0010591138852760196,0.0030667362734675407,0.006546072196215391,0.0035180094247334636 +fresh-zh-1,14,15,JS,strong_half,200,0.012686457112431526,0.03665739856660366,0.07552775144577026,0.04212494903244078,200,0.01816696673631668,0.03420329838991165,0.06253209970891475,0.038980009176302705 +fresh-zh-1,14,15,JS,random_half,200,0.005424338625743985,0.019764550030231476,0.044993088394403455,0.02369701297429856,200,0.0070193607825785875,0.017201948910951614,0.04024641029536724,0.021719297642703167 +fresh-zh-1,14,15,JS,all_heads,200,0.012738580256700516,0.03808196075260639,0.07442388385534286,0.042630896390182894,200,0.01994530074298382,0.03742143139243126,0.06954159215092659,0.042498018585611134 +fresh-zh-1,14,15,sqrtJS,weak_half,200,0.03218586407601833,0.05010714568197727,0.0879971645772457,0.056197855416685344,200,0.08224926590919494,0.13719164580106735,0.1984311446547508,0.141430749502033 +fresh-zh-1,14,15,sqrtJS,strong_half,200,0.11263386309146882,0.191461019217968,0.27482214868068694,0.19490188354626298,200,0.32469537258148196,0.4419621080160141,0.6150701522827148,0.4569437711685896 +fresh-zh-1,14,15,sqrtJS,random_half,200,0.07365011051297188,0.14058026671409607,0.21211453527212143,0.14319414671510458,200,0.20034082531929018,0.2910020500421524,0.4369297593832016,0.3076332885399461 +fresh-zh-1,14,15,sqrtJS,all_heads,200,0.11286516487598419,0.1951458603143692,0.2728074163198471,0.19614949336275458,200,0.4329228550195694,0.5795048475265503,0.8028445780277252,0.5983745224773884 +fresh-zh-1,15,16,Q,weak_half,400,0.038857254385948184,0.09299380704760551,0.18612721860408785,0.10176194860599935,400,0.07654964700341227,0.17181213945150375,0.29478941261768343,0.18026274639181794 +fresh-zh-1,15,16,Q,strong_half,400,0.08267682418227199,0.18284673988819122,0.3311071336269379,0.20537669679149986,400,0.1261170119047165,0.2957427203655243,0.6176675021648407,0.32882154604420066 +fresh-zh-1,15,16,Q,random_half,400,0.053788205981254576,0.13252561539411545,0.2669716328382492,0.1444959375075996,400,0.10495245531201366,0.22599444538354874,0.43323204517364505,0.2520154656469822 +fresh-zh-1,15,16,Q,all_heads,400,0.10741739571094513,0.22130366414785385,0.40556448698043823,0.24233471377752722,400,0.19986069649457933,0.46389444172382355,0.895671045780182,0.5090842915885151 +fresh-zh-1,15,16,K,weak_half,400,0.056583832949399955,0.10458607971668243,0.17701809108257296,0.11377417602576316,400,0.10486635565757751,0.17193111777305603,0.25912173688411716,0.17851479639299214 +fresh-zh-1,15,16,K,strong_half,400,0.10179118514060974,0.16995204240083694,0.241888764500618,0.1722904297336936,400,0.17079311609268188,0.29014453291893005,0.4582800298929215,0.303629362732172 +fresh-zh-1,15,16,K,random_half,400,0.07084484621882439,0.13180510699748993,0.23284293562173847,0.14696850360371172,400,0.13205492198467256,0.22686012089252472,0.361565461754799,0.23860157769173385 +fresh-zh-1,15,16,K,all_heads,400,0.13065983057022096,0.2146652266383171,0.327623051404953,0.22070790637284518,400,0.286562505364418,0.46163269877433777,0.715995854139328,0.48214415926486254 +fresh-zh-1,15,16,V,weak_half,400,0.05592040717601776,0.09025399759411812,0.17007001340389258,0.10424631589092315,400,0.10316894724965096,0.14961794018745422,0.23688530176877975,0.1625222942698747 +fresh-zh-1,15,16,V,strong_half,400,0.11136132031679154,0.17553451657295227,0.2472646623849869,0.17744086031801998,400,0.19820318073034288,0.26298415660858154,0.4204308032989502,0.2911523127928376 +fresh-zh-1,15,16,V,random_half,400,0.07574589997529983,0.13013328611850739,0.22329605668783192,0.1423272186797112,400,0.13742980659008025,0.20691635459661484,0.3400592684745789,0.22375178340822457 +fresh-zh-1,15,16,V,all_heads,400,0.16441787630319596,0.2226339653134346,0.3241157323122025,0.23453418442979454,400,0.31015911102294924,0.41454872488975525,0.6638900160789489,0.4536746100336313 +fresh-zh-1,15,16,QKV,weak_half,400,0.06382351741194725,0.09970645606517792,0.17476700693368913,0.11046133138239383,400,0.12111025005578996,0.1741028130054474,0.2554803878068924,0.18084673434495926 +fresh-zh-1,15,16,QKV,strong_half,400,0.12663210779428483,0.18337640166282654,0.2633481413125992,0.19299012422561646,400,0.20918588638305666,0.2980833500623703,0.49674830436706546,0.32359335504472253 +fresh-zh-1,15,16,QKV,random_half,400,0.08670373037457466,0.1383349522948265,0.2346001535654068,0.1505389334168285,400,0.1559004604816437,0.23158391565084457,0.3691472470760346,0.24929768467321992 +fresh-zh-1,15,16,QKV,all_heads,400,0.1660640448331833,0.22918907552957535,0.33692508935928345,0.24112250573933125,400,0.33140948712825774,0.47649067640304565,0.7405438065528871,0.5044400925189256 +fresh-zh-1,16,17,Q,weak_half,400,0.011443487741053105,0.03147801198065281,0.06484831720590592,0.03525240204297006,400,0.028641684725880624,0.08828307688236237,0.17327176332473757,0.09632125392090529 +fresh-zh-1,16,17,Q,strong_half,400,0.020274695940315727,0.07564282417297363,0.15376557260751725,0.08221106169279664,400,0.05904820077121258,0.20677544176578522,0.4035345584154129,0.22533540974371136 +fresh-zh-1,16,17,Q,random_half,400,0.016193052008748057,0.05296829715371132,0.1200816109776497,0.061771428841166195,400,0.047868327051401136,0.1443091779947281,0.2975869953632356,0.16144930059555918 +fresh-zh-1,16,17,Q,all_heads,400,0.02198704779148102,0.08053431287407875,0.1681451261043549,0.08844767664559186,400,0.08711719885468483,0.29948510229587555,0.5730169951915742,0.32165666183456776 +fresh-zh-1,16,17,K,weak_half,400,0.033124403282999995,0.050165845081210136,0.07849899157881737,0.05337870463728905,400,0.09348574355244636,0.12919239699840546,0.18416673541069034,0.13469661058858037 +fresh-zh-1,16,17,K,strong_half,400,0.07868001759052277,0.1108652837574482,0.1587536171078682,0.1151847325079143,400,0.22436089515686036,0.2992419898509979,0.41926400959491733,0.3125208915770054 +fresh-zh-1,16,17,K,random_half,400,0.056143035367131236,0.08373206853866577,0.12324093431234363,0.08837788569740951,400,0.15244996249675752,0.21413296461105347,0.3141970038414002,0.2252408615499735 +fresh-zh-1,16,17,K,all_heads,400,0.08364789113402367,0.1191946305334568,0.16787991374731065,0.1230351194832474,400,0.3286965906620026,0.4223298579454422,0.5934893786907196,0.44721750292927026 +fresh-zh-1,16,17,V,weak_half,400,0.02706148996949196,0.04358627088367939,0.07056583985686302,0.04675015435786918,400,0.07343796342611314,0.11341721192002296,0.1645933210849762,0.11798137019388377 +fresh-zh-1,16,17,V,strong_half,400,0.07018189653754235,0.10717492178082466,0.16017392426729207,0.1119536845665425,400,0.2089487463235855,0.2946387827396393,0.4093550473451614,0.30355792339891197 +fresh-zh-1,16,17,V,random_half,400,0.0511666152626276,0.07509138435125351,0.11617883518338207,0.08232155928388238,400,0.135181225836277,0.2040977105498314,0.2909060537815094,0.2122734265215695 +fresh-zh-1,16,17,V,all_heads,400,0.07686776369810104,0.11788614094257355,0.16847684234380722,0.12135644031688571,400,0.29818210005760193,0.41069212555885315,0.556501829624176,0.4215392939373851 +fresh-zh-1,16,17,QKV,weak_half,400,0.03445380106568337,0.04626351781189442,0.06468306928873062,0.047912679151631894,400,0.08933161944150925,0.1222173199057579,0.16380601823329927,0.12442634698003531 +fresh-zh-1,16,17,QKV,strong_half,400,0.0798524796962738,0.10440678894519806,0.14284701794385912,0.10905168137513101,400,0.22694631665945053,0.2960457354784012,0.3764394521713257,0.301054704003036 +fresh-zh-1,16,17,QKV,random_half,400,0.05802265554666519,0.07936752215027809,0.11070759743452073,0.08218498495407403,400,0.1519860327243805,0.20644665509462357,0.2840970605611801,0.21385112147778274 +fresh-zh-1,16,17,QKV,all_heads,400,0.0876778744161129,0.11382214725017548,0.15216546654701235,0.11745381616055965,400,0.32247751355171206,0.41705241799354553,0.5280065894126892,0.42548105120658875 +fresh-zh-1,17,18,Q,weak_half,400,0.010257887095212937,0.02985017653554678,0.05346549525856972,0.03076530867256224,400,0.02909612637013197,0.0822359211742878,0.14109483659267427,0.08341844645095989 +fresh-zh-1,17,18,Q,strong_half,400,0.03644267357885838,0.11225827783346176,0.18849411308765412,0.11239456389565021,400,0.08125411123037339,0.24294082820415497,0.42012641727924355,0.24995264974422754 +fresh-zh-1,17,18,Q,random_half,400,0.01930955052375794,0.0643124133348465,0.13231089413166053,0.07200149241369218,400,0.0482026256620884,0.15550408512353897,0.2984216809272766,0.16490474955178797 +fresh-zh-1,17,18,Q,all_heads,400,0.037206691503524784,0.11614489182829857,0.20175895392894747,0.11821210158523172,400,0.1144464284181595,0.3289668709039688,0.5647181928157806,0.3333710967283696 +fresh-zh-1,17,18,K,weak_half,400,0.015244142897427083,0.02842588070780039,0.04509932771325113,0.02964903718326241,400,0.04358677715063095,0.07665429636836052,0.10999363809823992,0.07711731424555182 +fresh-zh-1,17,18,K,strong_half,400,0.05522463619709015,0.09511182829737663,0.15276232063770298,0.10028549429494887,400,0.13043241798877717,0.22156314551830292,0.3179012954235078,0.224539214335382 +fresh-zh-1,17,18,K,random_half,400,0.03322337046265602,0.062268609181046486,0.11329310536384583,0.069703313075006,400,0.08056338280439376,0.14012230932712555,0.2373232662677765,0.14943318227306007 +fresh-zh-1,17,18,K,all_heads,400,0.058018385991454126,0.10291941836476326,0.16269583851099015,0.10671613374724984,400,0.1714495748281479,0.30153246223926544,0.42404144108295444,0.30165652837604284 +fresh-zh-1,17,18,V,weak_half,400,0.017294227331876754,0.027432105503976345,0.043192121759057046,0.029055988872423767,400,0.05025103576481342,0.07533105462789536,0.10334931239485742,0.07601329230237752 +fresh-zh-1,17,18,V,strong_half,400,0.0928646445274353,0.12618117779493332,0.171262988448143,0.13070686372928322,400,0.22375009804964066,0.28060898184776306,0.35995116233825686,0.285943387132138 +fresh-zh-1,17,18,V,random_half,400,0.037820407003164296,0.0777093879878521,0.13685556650161743,0.08445335913449525,400,0.09271823763847352,0.1736852526664734,0.2652476996183396,0.1781475749798119 +fresh-zh-1,17,18,V,all_heads,400,0.09430781602859498,0.1281353160738945,0.17362426817417148,0.13361172402277588,400,0.2788433939218521,0.35655370354652405,0.4507291376590729,0.361956678815186 +fresh-zh-1,17,18,QKV,weak_half,400,0.021323871798813343,0.030228035524487495,0.042895754799246795,0.03137299460358918,400,0.058929950743913655,0.08217205107212067,0.10875751227140429,0.08318613479845226 +fresh-zh-1,17,18,QKV,strong_half,400,0.08291090056300163,0.11636317893862724,0.1607620656490326,0.11987902893684804,400,0.20242394357919694,0.2641659677028656,0.3509768158197404,0.2709306503459811 +fresh-zh-1,17,18,QKV,random_half,400,0.04388960897922516,0.07484377175569534,0.1260247379541397,0.08012503677513451,400,0.1018272303044796,0.16714951395988464,0.2608159929513932,0.1750102079473436 +fresh-zh-1,17,18,QKV,all_heads,400,0.08587519377470017,0.12112954258918762,0.1683414444327355,0.12511482609435917,400,0.26694580614566804,0.34472839534282684,0.4579548001289368,0.354116785787046 +fresh-zh-1,18,19,Q,weak_half,200,0.04418631196022034,0.05085578374564648,0.05837373472750187,0.05097706327214837,200,0.12090492397546769,0.13881733268499374,0.15969208478927613,0.13879961650818587 +fresh-zh-1,18,19,Q,strong_half,200,0.11053699851036072,0.13854257762432098,0.18733840435743332,0.1424497913569212,200,0.27506838738918304,0.3293113112449646,0.40446606278419495,0.33607237718999383 +fresh-zh-1,18,19,Q,random_half,200,0.06894460618495941,0.10127710551023483,0.141543947160244,0.10444585120305419,200,0.1839703291654587,0.24141164124011993,0.3101217895746231,0.24563235245645046 +fresh-zh-1,18,19,Q,all_heads,200,0.12058041170239449,0.14813663065433502,0.188666532933712,0.15040689524263143,200,0.40186614394187925,0.46604764461517334,0.5630262672901154,0.47487199619412424 +fresh-zh-1,18,19,K,weak_half,200,0.021170202642679214,0.027108877897262573,0.03920099884271622,0.028381400033831595,200,0.05695937015116215,0.0693107545375824,0.10068894922733307,0.07280932968482375 +fresh-zh-1,18,19,K,strong_half,200,0.07167170196771622,0.08311523497104645,0.13250362724065778,0.09348452869802713,200,0.17832646518945694,0.21315638720989227,0.30426526069641113,0.22522490754723548 +fresh-zh-1,18,19,K,random_half,200,0.04234820045530796,0.0641087181866169,0.0996092140674591,0.06888951084576547,200,0.10071432515978813,0.1446828544139862,0.205939182639122,0.15174430243670942 +fresh-zh-1,18,19,K,all_heads,200,0.07549655064940453,0.0875764787197113,0.1375073090195656,0.09934411577880382,200,0.23835578858852388,0.2800897806882858,0.4034780204296112,0.2980342385172844 +fresh-zh-1,18,19,V,weak_half,200,0.02686607390642166,0.0368012972176075,0.07150087878108025,0.04140368173830211,200,0.06389357522130013,0.08974499255418777,0.19554361402988432,0.10402941972017288 +fresh-zh-1,18,19,V,strong_half,200,0.07389224320650102,0.10260581970214844,0.19325224161148072,0.11520451763644815,200,0.14788609743118286,0.18603487312793732,0.41192199885845165,0.22479225270450115 +fresh-zh-1,18,19,V,random_half,200,0.039930160716176034,0.0754949301481247,0.1307567611336708,0.08331849345006048,200,0.09392056688666345,0.14486253261566162,0.29639693498611447,0.16821642596274614 +fresh-zh-1,18,19,V,all_heads,200,0.08880241438746453,0.12536270916461945,0.21007540673017494,0.13627349704504013,200,0.21056183874607087,0.2666453719139099,0.6062463045120234,0.3288216692209244 +fresh-zh-1,18,19,QKV,weak_half,200,0.03369736596941948,0.04050724767148495,0.05336896777153015,0.04207921584136784,200,0.09036844670772552,0.10614202171564102,0.14560694694519044,0.11191326446831226 +fresh-zh-1,18,19,QKV,strong_half,200,0.08815728649497032,0.11350367590785027,0.16189422160387038,0.12053169049322605,200,0.2164504885673523,0.2655373811721802,0.36151763200759884,0.28128469310700893 +fresh-zh-1,18,19,QKV,random_half,200,0.057051916420459745,0.08637253940105438,0.12231793627142906,0.08857718460261822,200,0.14538584798574447,0.19379165768623352,0.2791553884744644,0.20188897661864758 +fresh-zh-1,18,19,QKV,all_heads,200,0.09985854178667068,0.123726736754179,0.16991755515336984,0.13238748345524073,200,0.3160225242376328,0.37049320340156555,0.5122994422912598,0.39319795683026315 +fresh-zh-1,18,19,JS,weak_half,200,0.00018686174153117465,0.0003997726598754525,0.0008203880803193896,0.00047592474653356475,200,0.0003759796207305044,0.0009927094215527177,0.011137682106345892,0.003672814778983593 +fresh-zh-1,18,19,JS,strong_half,200,0.0048529275227338076,0.00955058354884386,0.016848956234753132,0.010534159516100771,200,0.0036859582643955953,0.008272666949778795,0.016817902028560636,0.009128063932294025 +fresh-zh-1,18,19,JS,random_half,200,0.0030644220765680074,0.005775638623163104,0.010964615829288957,0.0065039870713371785,200,0.0014916536398231984,0.005064554279670119,0.012322360649704933,0.006195224224647973 +fresh-zh-1,18,19,JS,all_heads,200,0.005391384940594435,0.010478835552930832,0.019911780953407284,0.011639956755097956,200,0.0073930966202169655,0.012181064113974571,0.018965316005051136,0.012800878721754998 +fresh-zh-1,18,19,sqrtJS,weak_half,200,0.013669670838862658,0.019994291476905346,0.02864226140081882,0.020835379902273417,200,0.0504544023424387,0.07672679796814919,0.14746575951576232,0.09108220683410764 +fresh-zh-1,18,19,sqrtJS,strong_half,200,0.06966186612844467,0.09772692620754242,0.12980352640151976,0.0996908407099545,200,0.14282488375902175,0.20286782830953598,0.26792998909950255,0.20413716487586497 +fresh-zh-1,18,19,sqrtJS,random_half,200,0.05535721406340599,0.07599762082099915,0.10471140667796135,0.07803586592897772,200,0.09312209337949753,0.14217882603406906,0.20626243352890014,0.14802259024232625 +fresh-zh-1,18,19,sqrtJS,all_heads,200,0.07342604845762253,0.10236614942550659,0.14110672771930693,0.10465967688709497,200,0.2433579832315445,0.29331015050411224,0.3536794751882553,0.2952193719148636 +fresh-zh-1,19,20,Q,weak_half,400,0.01807730346918106,0.043068816885352135,0.08061934933066368,0.046873385389335456,400,0.03592093102633953,0.08650193735957146,0.147806690633297,0.0886238675750792 +fresh-zh-1,19,20,Q,strong_half,400,0.10447448864579201,0.18862545490264893,0.3570908159017563,0.22232691440731286,400,0.11946913078427315,0.24827250093221664,0.5088837385177615,0.2879188606515527 +fresh-zh-1,19,20,Q,random_half,400,0.04948151707649231,0.1076146699488163,0.22027959525585178,0.12904853399842978,400,0.0702143467962742,0.1580723449587822,0.32852236926555634,0.1859076172951609 +fresh-zh-1,19,20,Q,all_heads,400,0.12281462401151658,0.21579932421445847,0.39924546480178835,0.24608907375484704,400,0.15525705516338348,0.3394973576068878,0.6571131944656374,0.376542727611959 +fresh-zh-1,19,20,K,weak_half,400,0.026928113400936128,0.04720720276236534,0.07668921947479249,0.05047876248601824,400,0.05793135762214661,0.09058161452412605,0.1439066395163536,0.09529245444573461 +fresh-zh-1,19,20,K,strong_half,400,0.16089287549257278,0.27206967771053314,0.4463126987218857,0.28417910035699606,400,0.19896342754364013,0.2978910058736801,0.507819265127182,0.32737555842846633 +fresh-zh-1,19,20,K,random_half,400,0.07985168620944023,0.13445213437080383,0.27397170662879944,0.1580002803169191,400,0.11115921214222908,0.1822100430727005,0.3349448323249817,0.20742274487391113 +fresh-zh-1,19,20,K,all_heads,400,0.18001823425292968,0.2966523766517639,0.4930734544992447,0.3173040004074574,400,0.26754088401794435,0.3827636390924454,0.6346318840980529,0.42266801200807097 +fresh-zh-1,19,20,V,weak_half,400,0.026026525534689426,0.044772541150450706,0.07310919761657715,0.048153640232048926,400,0.051160100102424624,0.08251660689711571,0.13949311375617984,0.09140863832086325 +fresh-zh-1,19,20,V,strong_half,400,0.166020405292511,0.23247678577899933,0.31835949420928955,0.2388512161374092,400,0.21908978521823883,0.28261858224868774,0.385750761628151,0.293313883729279 +fresh-zh-1,19,20,V,random_half,400,0.0725831188261509,0.13175398111343384,0.2153692588210106,0.14152943075168878,400,0.10411842614412309,0.17992262542247772,0.27852448821067816,0.19102506749331952 +fresh-zh-1,19,20,V,all_heads,400,0.17581839710474015,0.26367752254009247,0.3710204243659973,0.2691542476043105,400,0.27741946578025817,0.3646242618560791,0.5320786833763124,0.38472252249717714 +fresh-zh-1,19,20,QKV,weak_half,400,0.030978120490908624,0.04920877702534199,0.07124503999948502,0.05086899711750448,400,0.061890507489442824,0.09332239627838135,0.13469395935535433,0.09651903699152171 +fresh-zh-1,19,20,QKV,strong_half,400,0.1753857046365738,0.2520713806152344,0.3637046247720718,0.26020050909370185,400,0.2259402394294739,0.3035884350538254,0.4514595806598664,0.3221036724373698 +fresh-zh-1,19,20,QKV,random_half,400,0.08813020735979081,0.1391705647110939,0.2351908713579178,0.15098282110877334,400,0.12285896092653274,0.1953536793589592,0.32359875738620764,0.20662547832354902 +fresh-zh-1,19,20,QKV,all_heads,400,0.192828269302845,0.28115274012088776,0.4051977902650833,0.2887783677875996,400,0.29529009461402894,0.4008778929710388,0.5732317984104156,0.41862270995974543 +fresh-zh-1,20,21,Q,weak_half,400,0.009367176797240973,0.016770681366324425,0.04343858473002912,0.021373582638334485,400,0.02442310694605112,0.04495573788881302,0.11610792279243475,0.05801464266842231 +fresh-zh-1,20,21,Q,strong_half,400,0.027276626974344253,0.04988645762205124,0.14450961947441102,0.06821255588904024,400,0.06541470438241959,0.11158574372529984,0.36110991537570963,0.1555602867435664 +fresh-zh-1,20,21,Q,random_half,400,0.01690274812281132,0.032926855608820915,0.09950623214244843,0.04588588968617842,400,0.04173469990491867,0.0785001739859581,0.22551043033599863,0.10648532232735305 +fresh-zh-1,20,21,Q,all_heads,400,0.029185904748737813,0.05292321369051933,0.15379064381122595,0.07220995042007416,400,0.09100260287523271,0.15788038820028305,0.4767947614192966,0.21357492849230766 +fresh-zh-1,20,21,K,weak_half,400,0.015485926717519761,0.02992810122668743,0.048041834682226184,0.03147126469993964,400,0.04581136293709278,0.08113198354840279,0.11337583810091023,0.08274268267210573 +fresh-zh-1,20,21,K,strong_half,400,0.047443211078643806,0.0986289344727993,0.14625866711139682,0.10217482417356223,400,0.11739995181560517,0.22006520628929138,0.30636286437511445,0.22804923102259636 +fresh-zh-1,20,21,K,random_half,400,0.031012327782809736,0.06877560168504715,0.10302622094750405,0.0707181069557555,400,0.07819945961236954,0.15078648179769516,0.21724917739629748,0.15428430399857462 +fresh-zh-1,20,21,K,all_heads,400,0.053820565342903144,0.10608701035380363,0.15607463270425798,0.11006549384444952,400,0.16491668224334718,0.30600596964359283,0.41316686868667607,0.3107919125072658 +fresh-zh-1,20,21,V,weak_half,400,0.0187362227588892,0.02855098433792591,0.04856846332550049,0.03189842443214729,400,0.053051631525158884,0.07432925328612328,0.13021092712879181,0.08319983382709324 +fresh-zh-1,20,21,V,strong_half,400,0.06175529733300209,0.09890245646238327,0.15515118986368182,0.10457868968136609,400,0.16760628670454025,0.2290603369474411,0.3217656701803208,0.23846623841673137 +fresh-zh-1,20,21,V,random_half,400,0.03662795163691044,0.06448699906468391,0.11022165045142175,0.07036696630064398,400,0.09709440320730209,0.14757636189460754,0.23738273233175278,0.15813218228518963 +fresh-zh-1,20,21,V,all_heads,400,0.06571206077933311,0.10736174881458282,0.17000610977411273,0.11368780387565494,400,0.2257467821240425,0.30166251957416534,0.4484910160303116,0.32166607152670623 +fresh-zh-1,20,21,QKV,weak_half,400,0.018704519793391227,0.029014505445957184,0.04211643747985363,0.02987536456435919,400,0.05282410643994808,0.07606765255331993,0.11316540166735649,0.07926684871315956 +fresh-zh-1,20,21,QKV,strong_half,400,0.06678980737924575,0.09101855009794235,0.13278564065694812,0.09765544516965746,400,0.15907405614852904,0.20604605972766876,0.3114944249391556,0.22341963682323696 +fresh-zh-1,20,21,QKV,random_half,400,0.03880481459200382,0.06436295434832573,0.09770176634192466,0.06693887199740857,400,0.09087926968932152,0.14062315970659256,0.2156922221183777,0.14970672152005135 +fresh-zh-1,20,21,QKV,all_heads,400,0.07206378653645516,0.09796501323580742,0.1411605075001717,0.10537502273917199,400,0.215184985101223,0.2814920246601105,0.41662318408489235,0.302686485722661 +fresh-zh-1,21,22,Q,weak_half,400,0.008660503942519427,0.01918204966932535,0.03235510438680649,0.019780594619223848,400,0.022406545653939246,0.05318986251950264,0.08965182304382326,0.055212842491455374 +fresh-zh-1,21,22,Q,strong_half,400,0.01710582785308361,0.039619917050004005,0.10278759822249416,0.0513183255540207,400,0.04095581471920014,0.10425002127885818,0.230678303539753,0.11881012118421495 +fresh-zh-1,21,22,Q,random_half,400,0.012858804035931827,0.028842171654105186,0.05764603018760681,0.03373508226592094,400,0.032151422649621966,0.07964494824409485,0.1399203673005104,0.08502164154779165 +fresh-zh-1,21,22,Q,all_heads,400,0.021914006024599076,0.0478077158331871,0.11429975926876076,0.059305494886357335,400,0.06380738615989685,0.15770221501588821,0.32294077277183536,0.1740229634847492 +fresh-zh-1,21,22,K,weak_half,400,0.011308252345770597,0.026440376415848732,0.044919623807072644,0.027582339709624648,400,0.03350218944251537,0.06986454129219055,0.11221962049603464,0.07138949180953205 +fresh-zh-1,21,22,K,strong_half,400,0.02613908890634775,0.054316723719239235,0.08673228845000268,0.055163874954450874,400,0.06248797550797463,0.1369437798857689,0.22195073664188386,0.14281931486912072 +fresh-zh-1,21,22,K,random_half,400,0.019276528246700765,0.04239805229008198,0.07223859503865243,0.045087054742034526,400,0.04739379994571209,0.10486841201782227,0.16931010633707047,0.1068450066121295 +fresh-zh-1,21,22,K,all_heads,400,0.030609668977558614,0.06241726502776146,0.09307577013969422,0.06232331372331828,400,0.09741592407226564,0.21235809475183487,0.33288378417491915,0.2142088066972792 +fresh-zh-1,21,22,V,weak_half,400,0.015414758026599884,0.023893700912594795,0.044169672951102266,0.02728994787670672,400,0.04159293584525585,0.0616496242582798,0.10178547129034998,0.06835457339417189 +fresh-zh-1,21,22,V,strong_half,400,0.034864951297640806,0.061660973355174065,0.08596278056502342,0.060664349682629105,400,0.08965963497757912,0.1361004114151001,0.21030150055885322,0.1426952115818858 +fresh-zh-1,21,22,V,random_half,400,0.023914260230958464,0.04297783970832825,0.07103102430701258,0.045497228153981266,400,0.061173249036073685,0.09642984718084335,0.15947003960609438,0.1053649883531034 +fresh-zh-1,21,22,V,all_heads,400,0.04160385131835938,0.07083310186862946,0.1005178935825825,0.07127607391215861,400,0.1340109333395958,0.19441517442464828,0.31463875174522404,0.21104978593066334 +fresh-zh-1,21,22,QKV,weak_half,400,0.015011560171842575,0.025508812628686428,0.03842748366296292,0.026152049545198678,400,0.042905883863568306,0.06812143325805664,0.09642958864569665,0.06845459720119834 +fresh-zh-1,21,22,QKV,strong_half,400,0.03424540609121323,0.05401398800313473,0.08542370796203615,0.059028689502738416,400,0.08846681788563729,0.14064470678567886,0.19629498273134233,0.14437464119866492 +fresh-zh-1,21,22,QKV,random_half,400,0.024949100613594056,0.04150465689599514,0.06324317753314972,0.043843034629244354,400,0.06614953204989434,0.10037439316511154,0.14740652292966844,0.10535573466680943 +fresh-zh-1,21,22,QKV,all_heads,400,0.040427465736866,0.0633632093667984,0.09504747539758686,0.06785769855137914,400,0.13258299827575684,0.2096063494682312,0.2849590599536896,0.21282923925668001 +fresh-zh-1,22,23,Q,weak_half,200,0.02660669069737196,0.03265906311571598,0.04163729026913643,0.0335950377676636,200,0.06976825520396233,0.08459222316741943,0.10750722438097,0.08762745048850774 +fresh-zh-1,22,23,Q,strong_half,200,0.09749124199151993,0.13140462338924408,0.17718934565782546,0.13404494989663363,200,0.2257361114025116,0.29010505974292755,0.36171768307685853,0.29375005453825 +fresh-zh-1,22,23,Q,random_half,200,0.051936833187937737,0.0830659493803978,0.13614254891872407,0.08880562009289861,200,0.12505317255854606,0.18164373934268951,0.2637925982475281,0.18803147468715906 +fresh-zh-1,22,23,Q,all_heads,200,0.10313952788710594,0.1351873204112053,0.1761267453432083,0.1380223059654236,200,0.3026728004217148,0.376559317111969,0.4655770808458328,0.38137750543653964 +fresh-zh-1,22,23,K,weak_half,200,0.021012074872851373,0.026431826874613762,0.03416493944823741,0.027471684883348644,200,0.052721232175827026,0.06949340179562569,0.08784109354019165,0.06940501123666763 +fresh-zh-1,22,23,K,strong_half,200,0.08454751968383789,0.13077029585838318,0.17503740787506103,0.1345434284582734,200,0.23189850747585297,0.29817962646484375,0.40552366375923155,0.30590287551283835 +fresh-zh-1,22,23,K,random_half,200,0.04253330565989018,0.08139684796333313,0.1538340136408806,0.09203396749682724,200,0.1007998064160347,0.1790110394358635,0.2858411461114883,0.18559118382632733 +fresh-zh-1,22,23,K,all_heads,200,0.08789847493171692,0.13170119374990463,0.17281343638896943,0.13520872771739958,200,0.28945009410381317,0.36516113579273224,0.4859142363071441,0.3753078833222389 +fresh-zh-1,22,23,V,weak_half,200,0.010867311619222166,0.014712155796587467,0.01853957772254944,0.014638461209833621,200,0.03031240589916706,0.039089299738407135,0.04691478498280048,0.03913877268321812 +fresh-zh-1,22,23,V,strong_half,200,0.05531908236443997,0.09389155358076096,0.15750364810228343,0.1040774910710752,200,0.15616471022367479,0.2066180631518364,0.32512587308883667,0.23288167864084244 +fresh-zh-1,22,23,V,random_half,200,0.025858629494905472,0.05438024923205376,0.14136562049388882,0.0696294843312353,200,0.06518338173627854,0.11331231519579887,0.24013014286756515,0.13227637842297554 +fresh-zh-1,22,23,V,all_heads,200,0.05541267655789852,0.08971264585852623,0.14975408613681787,0.10209212131798268,200,0.19312359988689423,0.2468925416469574,0.36598558723926544,0.27202045112848283 +fresh-zh-1,22,23,QKV,weak_half,200,0.021080661192536355,0.026230009272694588,0.03171968422830105,0.026593731390312313,200,0.055794306471943854,0.06829139962792397,0.08445351794362067,0.06954916277900339 +fresh-zh-1,22,23,QKV,strong_half,200,0.09464239552617074,0.12529269605875015,0.1578635111451149,0.12796436812728643,200,0.23224322646856307,0.2923937141895294,0.3521237939596176,0.292328168079257 +fresh-zh-1,22,23,QKV,random_half,200,0.04551886692643166,0.08006689324975014,0.1316432774066925,0.0868149936478585,200,0.10879526361823082,0.17011704295873642,0.262914177775383,0.17840492397546767 +fresh-zh-1,22,23,QKV,all_heads,200,0.09703452214598655,0.1274627298116684,0.15732169300317764,0.1289096276089549,200,0.29037469029426577,0.3600784242153168,0.4287635385990143,0.3618773307651281 +fresh-zh-1,22,23,JS,weak_half,200,5.494983224707541e-06,0.00013325523468665779,0.0003206852590665221,0.00015415791268537759,200,9.106294874072773e-06,0.0001708330964902416,0.0005779539525974542,0.00037931956416969114 +fresh-zh-1,22,23,JS,strong_half,200,0.00033902769209817054,0.003970096353441477,0.009359128214418885,0.004748657431082393,200,0.0006321087887044996,0.0046155673917382956,0.00844416469335556,0.004549483851587865 +fresh-zh-1,22,23,JS,random_half,200,0.00024059269198914993,0.0038516283966600895,0.009409911464899777,0.004397634998786089,200,0.0002455743131577038,0.001641014649067074,0.005733241513371468,0.002489293375356283 +fresh-zh-1,22,23,JS,all_heads,200,0.0003525617939885706,0.004056227160617709,0.009458037465810776,0.0047288248687254964,200,0.0006433107075281442,0.005029468098655343,0.00932961432263255,0.004928803415823495 +fresh-zh-1,22,23,sqrtJS,weak_half,200,0.002343594678677619,0.011543620843440294,0.017907664366066455,0.010906439796090127,200,0.0075754469726234674,0.03263021446764469,0.050667811557650565,0.031955912911798806 +fresh-zh-1,22,23,sqrtJS,strong_half,200,0.018412666767835616,0.06300868093967438,0.09674020633101461,0.06195419333409518,200,0.05459912233054638,0.15017910301685333,0.2018107458949089,0.13967236813157796 +fresh-zh-1,22,23,sqrtJS,random_half,200,0.015510733425617219,0.06206027418375015,0.09700406566262244,0.059201172315515575,200,0.02665636446326971,0.08175721392035484,0.1470184475183487,0.08546774195507169 +fresh-zh-1,22,23,sqrtJS,all_heads,200,0.018776021338999273,0.0636826902627945,0.09725243225693703,0.062189759551547465,200,0.06331494674086571,0.19330153614282608,0.2487553924322128,0.17162828089669346 +fresh-zh-2,0,1,Q,weak_half,400,0.2648376613855362,0.3874732702970505,0.5260235190391541,0.39343557808548213,400,0.6054875791072846,0.8713871836662292,1.1791668295860291,0.887275892496109 +fresh-zh-2,0,1,Q,strong_half,400,0.7214876234531403,1.0010417103767395,1.2982397317886354,1.0101331526786088,400,1.9194222688674927,2.6780645847320557,3.2674439191818236,2.637800497710705 +fresh-zh-2,0,1,Q,random_half,400,0.5363628029823303,0.9467560350894928,1.4078132748603822,0.9665903037041426,400,1.046249258518219,1.6389338374137878,2.5394014835357672,1.7456951531767846 +fresh-zh-2,0,1,Q,all_heads,400,0.9481504082679748,1.2277910113334656,1.4733830690383911,1.212661385089159,400,2.6784209966659547,3.5596717596054077,4.31563367843628,3.525076385140419 +fresh-zh-2,0,1,K,weak_half,400,0.2944824278354645,0.4314621835947037,0.6449759244918823,0.4662923963740468,400,0.6432840943336487,0.9595790803432465,1.3342329263687134,0.9866119316965342 +fresh-zh-2,0,1,K,strong_half,400,0.7177101671695709,0.9563775658607483,1.2509195804595947,0.9653528322279453,400,1.9928792357444762,2.563928246498108,3.430527758598328,2.669617845416069 +fresh-zh-2,0,1,K,random_half,400,0.5785976111888886,0.9141725599765778,1.2352944493293765,0.9141554401069879,400,1.1466878533363343,1.7445629835128784,2.543634653091431,1.814528128579259 +fresh-zh-2,0,1,K,all_heads,400,1.1080543756484986,1.2878097891807556,1.4697305202484132,1.2800560203939677,400,2.8104894399642943,3.57545006275177,4.577861118316651,3.656229771077633 +fresh-zh-2,0,1,V,weak_half,400,0.1628776118159294,0.3487873822450638,0.6365173399448395,0.3879943352378905,400,0.40911180078983306,0.7459304332733154,1.140502738952637,0.774768868200481 +fresh-zh-2,0,1,V,strong_half,400,0.4422972977161408,0.7783006131649017,1.3597102403640746,0.8558747356012464,400,1.2627275943756104,2.123578667640686,3.6950094223022467,2.337604142278433 +fresh-zh-2,0,1,V,random_half,400,0.3910761117935181,0.7526727914810181,1.632265305519104,0.9649375338852405,400,0.7758161664009094,1.4003964066505432,2.459721064567566,1.5493761852383614 +fresh-zh-2,0,1,V,all_heads,400,0.6325876295566559,0.9742241203784943,1.460292673110962,1.0494256625324487,400,1.7271244525909424,2.879338026046753,4.668634557723999,3.112373009324074 +fresh-zh-2,0,1,QKV,weak_half,400,0.31489810049533845,0.4117444157600403,0.561965423822403,0.43402250707149503,400,0.6953037202358247,0.9033407866954803,1.156979215145111,0.9254803396761417 +fresh-zh-2,0,1,QKV,strong_half,400,0.7486894249916077,0.9550988972187042,1.2130499362945557,0.9721889090538025,400,2.118990230560303,2.6693758964538574,3.3637067317962646,2.7054576340317724 +fresh-zh-2,0,1,QKV,random_half,400,0.6391725718975068,0.9338054955005646,1.3532358884811402,0.9987264425307513,400,1.1946433901786804,1.7519360184669495,2.507421851158142,1.8006141361594201 +fresh-zh-2,0,1,QKV,all_heads,400,1.017815065383911,1.1882756352424622,1.3918549180030824,1.2044357767701148,400,2.936536502838135,3.55461049079895,4.395965957641602,3.6309379708766936 +fresh-zh-2,1,2,Q,weak_half,400,0.03180607631802559,0.08617769926786423,0.17453702241182334,0.09723331775516271,400,0.07201119437813759,0.20979957282543182,0.4392178475856781,0.23527637095656245 +fresh-zh-2,1,2,Q,strong_half,400,0.1835287556052208,0.3922531306743622,0.677453088760376,0.4162833768315613,400,0.4064407736063004,0.9215261340141296,1.3860842347145081,0.9269571147486567 +fresh-zh-2,1,2,Q,random_half,400,0.11735428348183632,0.243020661175251,0.45796073377132424,0.27121559307910503,400,0.2492395579814911,0.5480234026908875,0.971581119298935,0.5805711693689227 +fresh-zh-2,1,2,Q,all_heads,400,0.20227622091770173,0.4206163287162781,0.6965594351291656,0.4398299381881952,400,0.5336476981639863,1.1550053358078003,1.7559188365936282,1.1622334879636764 +fresh-zh-2,1,2,K,weak_half,400,0.053262006118893625,0.1195601038634777,0.20003568977117542,0.13065170916728674,400,0.11578720733523369,0.26511794328689575,0.4699510902166368,0.28425284803844986 +fresh-zh-2,1,2,K,strong_half,400,0.27896004915237427,0.45271165668964386,0.722864317893982,0.48162702105939387,400,0.6944873571395874,1.0310574769973755,1.504101824760437,1.0891512825340033 +fresh-zh-2,1,2,K,random_half,400,0.17955655902624132,0.29841428995132446,0.5091466784477234,0.333430594317615,400,0.3561334490776062,0.6324241757392883,1.0587063431739807,0.6840360337495803 +fresh-zh-2,1,2,K,all_heads,400,0.29900295436382296,0.48416967689991,0.7730406105518342,0.5210956669226289,400,0.8472512722015381,1.2943285703659058,1.9201772093772889,1.3734041322767734 +fresh-zh-2,1,2,V,weak_half,400,0.028783070854842666,0.08696029707789421,0.19046647101640704,0.10560477533610538,400,0.07022152617573739,0.1951402723789215,0.43291854560375215,0.22895928781712427 +fresh-zh-2,1,2,V,strong_half,400,0.1637512192130089,0.35394585132598877,0.6668008625507356,0.39742026111111045,400,0.4714319497346878,0.8852927088737488,1.4191490292549136,0.916985050663352 +fresh-zh-2,1,2,V,random_half,400,0.11053737178444863,0.2308596968650818,0.4759960800409317,0.26900392265059053,400,0.2637630313634873,0.5102115869522095,0.9774772465229035,0.5721341818198562 +fresh-zh-2,1,2,V,all_heads,400,0.17112332582473755,0.3779972493648529,0.7181003987789154,0.4193329758010805,400,0.5713463187217712,1.0749849677085876,1.8424681544303894,1.1459443364292383 +fresh-zh-2,1,2,QKV,weak_half,400,0.05310108214616776,0.108100526034832,0.18913805037736892,0.11768706575036049,400,0.10931599363684655,0.24480511993169785,0.4520754337310791,0.2651156614162028 +fresh-zh-2,1,2,QKV,strong_half,400,0.26895219385623936,0.4324079751968384,0.6587443590164185,0.4508475965261459,400,0.6555113136768341,1.025124192237854,1.4239188313484192,1.04472472935915 +fresh-zh-2,1,2,QKV,random_half,400,0.17512062788009644,0.2834625095129013,0.4661668568849564,0.3066996150277555,400,0.37470181882381437,0.6130439043045044,0.9965296685695652,0.6541816806420684 +fresh-zh-2,1,2,QKV,all_heads,400,0.2923259794712067,0.46899132430553436,0.6892255187034607,0.4804381489381194,400,0.802663654088974,1.2894036173820496,1.841315007209778,1.3098403906822205 +fresh-zh-2,2,3,Q,weak_half,200,0.062620035931468,0.09841839969158173,0.15367291867733002,0.10408719444647431,200,0.13935784846544266,0.2287757769227028,0.37230240702629086,0.23862245585769415 +fresh-zh-2,2,3,Q,strong_half,200,0.40264391899108887,0.5243100225925446,0.6796107113361358,0.5336997556686401,200,0.7370131194591523,1.0067540407180786,1.3024721026420594,1.015425756275654 +fresh-zh-2,2,3,Q,random_half,200,0.17579802274703982,0.30586034059524536,0.4822785288095474,0.3200813458859921,200,0.3799231708049774,0.5952396392822266,0.9151497542858124,0.6216181799769401 +fresh-zh-2,2,3,Q,all_heads,200,0.4209339261054993,0.5569192171096802,0.714756828546524,0.5643459713459015,200,0.8993128776550293,1.2311509847640991,1.629216730594635,1.2540482077002526 +fresh-zh-2,2,3,K,weak_half,200,0.06379295811057091,0.10365629941225052,0.14335186183452603,0.1045555150695145,200,0.13798698782920837,0.23092404752969742,0.35366750955581666,0.2394820376485586 +fresh-zh-2,2,3,K,strong_half,200,0.39259134232997894,0.47367724776268005,0.5816201269626617,0.48099444180727,200,0.8160958528518677,1.062165081501007,1.1957700014114379,1.040636128783226 +fresh-zh-2,2,3,K,random_half,200,0.1935724914073944,0.3178025633096695,0.4247413456439972,0.3126764231175184,200,0.43104503452777865,0.6374415159225464,0.8681858539581299,0.6400616896152497 +fresh-zh-2,2,3,K,all_heads,200,0.4189120978116989,0.5081579089164734,0.5851820290088654,0.5072114789485931,200,1.0167563974857332,1.2891824841499329,1.536208415031433,1.2801181662082672 +fresh-zh-2,2,3,V,weak_half,200,0.08428309112787248,0.12304455041885376,0.18481378257274628,0.13194717787206173,200,0.176307313144207,0.3046201467514038,0.45224243104457834,0.30648116827011107 +fresh-zh-2,2,3,V,strong_half,200,0.4104331135749817,0.5091896951198578,0.6877521455287928,0.5372684827446937,200,0.9004841029644013,1.1179239749908447,1.6742329955101012,1.1826784741878509 +fresh-zh-2,2,3,V,random_half,200,0.21861404925584793,0.35159920156002045,0.5312645554542541,0.3657162569090724,200,0.4621188998222351,0.7000472545623779,1.0344832777976989,0.7409963382780552 +fresh-zh-2,2,3,V,all_heads,200,0.43679551482200624,0.5589534640312195,0.761405712366104,0.5876820468902588,200,1.0811328649520875,1.4131184816360474,2.1930981874465942,1.4891596412658692 +fresh-zh-2,2,3,QKV,weak_half,200,0.07225624769926071,0.11314621195197105,0.16195273995399476,0.11529422078281641,200,0.1553133726119995,0.25974881649017334,0.3992436110973358,0.2673435628414154 +fresh-zh-2,2,3,QKV,strong_half,200,0.4213111102581024,0.5087988972663879,0.6391741454601287,0.5211702813208103,200,0.8736815035343171,1.0528784394264221,1.3696529507637023,1.0967427599430084 +fresh-zh-2,2,3,QKV,random_half,200,0.19892132580280303,0.3248658627271652,0.4621700942516327,0.33641833063215015,200,0.4401358485221863,0.6618520617485046,0.9407800495624542,0.6792227746546269 +fresh-zh-2,2,3,QKV,all_heads,200,0.43919014632701875,0.544969916343689,0.6947742998600005,0.5576465930044651,200,1.0465959429740905,1.3263848423957825,1.8111236333847045,1.3640863183140755 +fresh-zh-2,2,3,JS,weak_half,200,0.0008754392911214381,0.002931673778221011,0.00838765287771821,0.003907600044549327,200,0.0016810386208817366,0.004821970127522945,0.010347506403923033,0.005693192623730283 +fresh-zh-2,2,3,JS,strong_half,200,0.03552554473280907,0.0734245739877224,0.13060719221830366,0.07953192181885242,200,0.029955943301320075,0.049833131954073906,0.08983037695288656,0.05435132003389299 +fresh-zh-2,2,3,JS,random_half,200,0.015027442574501039,0.041899025440216064,0.084806627035141,0.04657784797716886,200,0.008310757484287024,0.021570160053670406,0.05704445913434029,0.028791364133358 +fresh-zh-2,2,3,JS,all_heads,200,0.04212095141410828,0.08279650658369064,0.15481597185134885,0.09334839025512337,200,0.033448167517781256,0.05564773827791214,0.09787186905741692,0.060044512851163745 +fresh-zh-2,2,3,sqrtJS,weak_half,200,0.029587708786129954,0.05414476431906223,0.09158413112163544,0.057557814801111815,200,0.09828706458210945,0.17385190725326538,0.24827663153409957,0.1744667437672615 +fresh-zh-2,2,3,sqrtJS,strong_half,200,0.18848219960927964,0.2709682881832123,0.36139604449272156,0.2737042164802551,200,0.3857810080051422,0.5069295763969421,0.6742028832435607,0.5181437335163355 +fresh-zh-2,2,3,sqrtJS,random_half,200,0.12258537635207177,0.20469235628843307,0.2912157714366913,0.2067999693378806,200,0.22061512768268587,0.32048724591732025,0.49890652000904084,0.34085507404059173 +fresh-zh-2,2,3,sqrtJS,all_heads,200,0.2052334189414978,0.28774379193782806,0.3934634953737259,0.2956148480623961,200,0.5147837281227112,0.6829850971698761,0.8971190452575684,0.692610477656126 +fresh-zh-2,3,4,Q,weak_half,400,0.10182146951556206,0.18433339148759842,0.35674160420894624,0.20648589495569467,400,0.2040109381079674,0.3789905905723572,0.6319206118583681,0.4043818523362279 +fresh-zh-2,3,4,Q,strong_half,400,0.17408276051282884,0.3436475694179535,0.805352407693863,0.42082484358921646,400,0.4460858374834061,0.9474205374717712,2.2352325439453127,1.1501067965105176 +fresh-zh-2,3,4,Q,random_half,400,0.14974962323904037,0.30023303627967834,0.7069694578647621,0.3707930132932961,400,0.30886028707027435,0.6399976313114166,1.4378863930702213,0.7855646720528603 +fresh-zh-2,3,4,Q,all_heads,400,0.18556381464004518,0.3680010139942169,0.8319811880588531,0.4438873653486371,400,0.6812250912189484,1.347758710384369,2.874259328842163,1.5544886516779661 +fresh-zh-2,3,4,K,weak_half,400,0.14130924940109255,0.21221660822629929,0.3372112780809403,0.22705117797479032,400,0.28108370304107666,0.4041530638933182,0.613432002067566,0.42730597995221614 +fresh-zh-2,3,4,K,strong_half,400,0.22166369855403903,0.3345562666654587,0.587812238931656,0.37076287671923636,400,0.6246818363666534,0.9605116844177246,1.5894199967384344,1.0290297851711512 +fresh-zh-2,3,4,K,random_half,400,0.19788132905960085,0.3214886784553528,0.576952874660492,0.36219941910356285,400,0.4111116617918015,0.6665836274623871,1.0974230170249948,0.722776158824563 +fresh-zh-2,3,4,K,all_heads,400,0.24956860691308977,0.38758376240730286,0.6451585590839387,0.4216957419738174,400,0.9337062120437623,1.3523602485656738,2.156136465072632,1.4563357625901698 +fresh-zh-2,3,4,V,weak_half,400,0.08521197438240051,0.18105529993772507,0.3584404677152634,0.201632323525846,400,0.19453328251838684,0.3517194092273712,0.6226886212825775,0.38557370262220503 +fresh-zh-2,3,4,V,strong_half,400,0.1508300855755806,0.30937454104423523,0.5389402866363526,0.3311061461176723,400,0.49177001118659974,0.8877959251403809,1.6569042921066284,0.9960330358147621 +fresh-zh-2,3,4,V,random_half,400,0.12036197930574417,0.27332691848278046,0.530519837141037,0.3125325441546738,400,0.30945872962474824,0.6116665601730347,1.114799189567566,0.6857386314123869 +fresh-zh-2,3,4,V,all_heads,400,0.14503857493400574,0.3333306312561035,0.6077094614505768,0.3564210488833487,400,0.7077728331089019,1.2413044571876526,2.2631957530975346,1.3816067403554917 +fresh-zh-2,3,4,QKV,weak_half,400,0.134733285009861,0.2014886885881424,0.33525277376174933,0.21942302519455553,400,0.276588037610054,0.397394597530365,0.6060259759426118,0.4221608819812536 +fresh-zh-2,3,4,QKV,strong_half,400,0.2206121787428856,0.35293637216091156,0.6309982776641846,0.394196913652122,400,0.6410978972911835,1.0175457000732422,1.7717644453048706,1.1213500546664 +fresh-zh-2,3,4,QKV,random_half,400,0.18808986693620683,0.31979915499687195,0.6136504590511322,0.36762104207649826,400,0.39918387234210967,0.6878704130649567,1.2886569499969482,0.7714938297495246 +fresh-zh-2,3,4,QKV,all_heads,400,0.24418131411075591,0.38481272757053375,0.6817767918109894,0.4278284328430891,400,0.9467487215995789,1.4166229963302612,2.359205913543702,1.5435109341144562 +fresh-zh-2,4,5,Q,weak_half,400,0.06499623879790306,0.11138837411999702,0.16697186678647996,0.11569185595959425,400,0.187619411945343,0.3020610958337784,0.4326162219047547,0.3069116202928126 +fresh-zh-2,4,5,Q,strong_half,400,0.14592598825693132,0.24117906391620636,0.35668340921401975,0.24721363957971335,400,0.4338626146316529,0.7121109664440155,0.9709268927574158,0.7141183137148619 +fresh-zh-2,4,5,Q,random_half,400,0.10356688648462296,0.18423531204462051,0.2835604369640351,0.1951833973824978,400,0.2966772586107254,0.5036535561084747,0.7416207134723664,0.5148971027135849 +fresh-zh-2,4,5,Q,all_heads,400,0.15943476557731628,0.26932020485401154,0.38955158889293673,0.27421941006556155,400,0.6286894202232362,1.023394227027893,1.3528278589248657,1.0210299324989318 +fresh-zh-2,4,5,K,weak_half,400,0.07325488775968551,0.11947610974311829,0.1856792971491814,0.12589082837104798,400,0.19750196635723116,0.30144503712654114,0.41723301410675057,0.2993407387565821 +fresh-zh-2,4,5,K,strong_half,400,0.1402723401784897,0.23143436014652252,0.33935459256172185,0.23689203730784356,400,0.46981183886528016,0.6746941208839417,0.9649900257587434,0.6909499964118004 +fresh-zh-2,4,5,K,random_half,400,0.11062419265508652,0.18380456417798996,0.2810231149196625,0.19213083242066203,400,0.31478704512119293,0.4879765212535858,0.7200738430023194,0.49793915143236517 +fresh-zh-2,4,5,K,all_heads,400,0.16129197478294371,0.2604702562093735,0.3885503351688385,0.26804876609705386,400,0.6756592154502868,0.9881969690322876,1.3682786583900453,0.9902907348051667 +fresh-zh-2,4,5,V,weak_half,400,0.03411308787763119,0.08372387290000916,0.1888744369149208,0.09786862903274596,400,0.10203538984060288,0.21058041602373123,0.4024018615484238,0.2354505905136466 +fresh-zh-2,4,5,V,strong_half,400,0.06478551924228669,0.16193684190511703,0.3392485529184342,0.19384302674327045,400,0.22748850733041764,0.48807835578918457,0.9659050524234774,0.5473582885041833 +fresh-zh-2,4,5,V,random_half,400,0.0514755230396986,0.12855678796768188,0.277489048242569,0.1494828708935529,400,0.15864482820034026,0.34322743117809296,0.7215989589691164,0.3920198117941618 +fresh-zh-2,4,5,V,all_heads,400,0.07805272042751314,0.18469876050949097,0.38876969814300555,0.21871988544240595,400,0.3395502775907516,0.6936745345592499,1.35723181962967,0.7828088762983679 +fresh-zh-2,4,5,QKV,weak_half,400,0.07447807639837266,0.11285458877682686,0.17265537679195406,0.1187415668554604,400,0.19998560398817064,0.2864484637975693,0.40641095042228703,0.29688146628439427 +fresh-zh-2,4,5,QKV,strong_half,400,0.15238103717565538,0.22816289216279984,0.3307213008403778,0.23706520315259694,400,0.47056399881839756,0.6712685525417328,0.9729892492294312,0.6956351934373379 +fresh-zh-2,4,5,QKV,random_half,400,0.12187353223562242,0.18106570094823837,0.26290630102157597,0.1884796711988747,400,0.3305923521518707,0.4762432128190994,0.7106719791889193,0.4993589485436678 +fresh-zh-2,4,5,QKV,all_heads,400,0.1730109304189682,0.2522929906845093,0.37435882091522216,0.2651132765598595,400,0.6803235471248626,0.9590906500816345,1.3362675666809083,0.9925166627764702 +fresh-zh-2,5,6,Q,weak_half,400,0.09645748138427734,0.14302179962396622,0.20897110402584082,0.1482873978279531,400,0.2599897563457489,0.38999447226524353,0.5479434072971344,0.39855748232454064 +fresh-zh-2,5,6,Q,strong_half,400,0.1809565916657448,0.2579386979341507,0.4242433369159699,0.2866741880774498,400,0.5696765661239624,0.8262857496738434,1.1645463705062868,0.8499522567540407 +fresh-zh-2,5,6,Q,random_half,400,0.14023137986660003,0.21207524836063385,0.3358057290315628,0.2294808317348361,400,0.39617145955562594,0.603546679019928,0.8519620954990388,0.6227715906873346 +fresh-zh-2,5,6,Q,all_heads,400,0.2124523475766182,0.3114890903234482,0.48547130525112153,0.3315464476495981,400,0.8500074565410614,1.2207114696502686,1.6831544876098634,1.2485097397863865 +fresh-zh-2,5,6,K,weak_half,400,0.11268386095762253,0.15844296663999557,0.23073527663946153,0.16511732494458556,400,0.30841091871261594,0.4075973480939865,0.5671684145927429,0.42284399185329674 +fresh-zh-2,5,6,K,strong_half,400,0.18963371217250827,0.27547281980514526,0.39410830736160285,0.2861934428662062,400,0.5813629508018494,0.8026167452335358,1.1462677121162417,0.8330651656538248 +fresh-zh-2,5,6,K,random_half,400,0.15584196597337724,0.22868263721466064,0.3482868522405625,0.245213487111032,400,0.4267770618200302,0.599095344543457,0.8618950605392456,0.6257508716732264 +fresh-zh-2,5,6,K,all_heads,400,0.22099290490150453,0.32103320956230164,0.45870922803878783,0.3306591249629855,400,0.8992205858230591,1.2125543355941772,1.6854336857795715,1.2559091521799564 +fresh-zh-2,5,6,V,weak_half,400,0.07063206806778909,0.1404784992337227,0.2555223971605301,0.15623776616062968,400,0.20212573707103731,0.3495407998561859,0.568945848941803,0.3724437328800559 +fresh-zh-2,5,6,V,strong_half,400,0.12446456775069237,0.2431713342666626,0.4491926223039629,0.2683499524742365,400,0.43640015423297884,0.7116361856460571,1.2341303348541262,0.7784052428603172 +fresh-zh-2,5,6,V,random_half,400,0.09991363734006882,0.19938576221466064,0.37734529376029985,0.22433927159756423,400,0.31030079126358034,0.5269667208194733,0.9289289832115174,0.576730994656682 +fresh-zh-2,5,6,V,all_heads,400,0.15064573138952256,0.2901022881269455,0.5157874703407289,0.313900042232126,400,0.6542597651481629,1.0770132541656494,1.7728758096694948,1.1508489708602427 +fresh-zh-2,5,6,QKV,weak_half,400,0.11484371572732927,0.15857937932014465,0.21937501430511475,0.16239743519574404,400,0.3060603469610214,0.4139917939901352,0.5294583380222322,0.41633064359426497 +fresh-zh-2,5,6,QKV,strong_half,400,0.21021323651075363,0.2711845487356186,0.3885083347558975,0.2908404967188835,400,0.642687463760376,0.8447873890399933,1.1069746375083926,0.867037114277482 +fresh-zh-2,5,6,QKV,random_half,400,0.16933705806732177,0.2327672466635704,0.3332730144262314,0.2428640996478498,400,0.4467173606157303,0.6256259083747864,0.8484794855117799,0.6411347587406635 +fresh-zh-2,5,6,QKV,all_heads,400,0.23886077255010604,0.3242548555135727,0.44796579480171206,0.33672554697841406,400,0.9629941642284393,1.2636644840240479,1.6333376526832581,1.2833677542209625 +fresh-zh-2,6,7,Q,weak_half,200,0.0417391087859869,0.07232262939214706,0.11212609261274337,0.07467025758698582,200,0.10848366022109986,0.18776314705610275,0.28790284097194674,0.19288351770490408 +fresh-zh-2,6,7,Q,strong_half,200,0.25640063285827636,0.3177511990070343,0.4148035258054733,0.32744133576750756,200,0.5647796273231507,0.716688722372055,0.954166054725647,0.7400373034179211 +fresh-zh-2,6,7,Q,random_half,200,0.12879900634288788,0.22194886952638626,0.332408207654953,0.2256222191080451,200,0.28238392472267154,0.4688715189695358,0.6756695747375489,0.47186466835439206 +fresh-zh-2,6,7,Q,all_heads,200,0.269546315073967,0.33507607877254486,0.4514196515083313,0.348991626650095,200,0.6870147585868835,0.9055735170841217,1.2147770643234252,0.9329208162426949 +fresh-zh-2,6,7,K,weak_half,200,0.04985336102545261,0.07939891517162323,0.11380258202552794,0.08087593361735344,200,0.12708827108144763,0.20006117224693298,0.29327012300491334,0.20637835524976253 +fresh-zh-2,6,7,K,strong_half,200,0.24310618489980698,0.3176184445619583,0.37958794832229614,0.31405646473169324,200,0.5958089709281922,0.765109658241272,0.8961048722267151,0.7666196539998055 +fresh-zh-2,6,7,K,random_half,200,0.15496660768985748,0.2248191013932228,0.3013159543275833,0.22552063286304475,200,0.3283988356590271,0.500721275806427,0.6430460333824157,0.49004470720887183 +fresh-zh-2,6,7,K,all_heads,200,0.24664741456508638,0.3402111679315567,0.39364457428455346,0.331775923371315,200,0.7348533928394317,0.9835070967674255,1.1769101858139037,0.9729980158805848 +fresh-zh-2,6,7,V,weak_half,200,0.0661732066422701,0.10102368146181107,0.13809432089328766,0.10337994577363134,200,0.1726634055376053,0.26922808587551117,0.37525428235530844,0.26471279837191103 +fresh-zh-2,6,7,V,strong_half,200,0.32914761900901796,0.3666203022003174,0.4801380693912506,0.3861409205198288,200,0.7577693998813629,0.912235677242279,1.1484794020652769,0.9280247834324836 +fresh-zh-2,6,7,V,random_half,200,0.178908771276474,0.27492569386959076,0.35554607808589933,0.27435755301266906,200,0.40946572721004487,0.6027847826480865,0.7827392041683197,0.600053438693285 +fresh-zh-2,6,7,V,all_heads,200,0.33678316175937656,0.39904166758060455,0.5169395923614502,0.413426753282547,200,0.9429453730583192,1.2080861926078796,1.4355559825897215,1.1927375829219817 +fresh-zh-2,6,7,QKV,weak_half,200,0.05415573380887509,0.08810650557279587,0.12289200201630592,0.08794686414301395,200,0.14467986673116684,0.2311157062649727,0.31491124033927914,0.22632790356874466 +fresh-zh-2,6,7,QKV,strong_half,200,0.28490555584430693,0.3426387906074524,0.4147398978471756,0.3459416618198156,200,0.6692609429359436,0.8287835717201233,0.9677507996559143,0.825439482331276 +fresh-zh-2,6,7,QKV,random_half,200,0.1650654748082161,0.24507559835910797,0.3306679308414459,0.24474985890090464,200,0.37460424900054934,0.5417675375938416,0.6911063194274902,0.5301071310043335 +fresh-zh-2,6,7,QKV,all_heads,200,0.2974673330783844,0.3694232851266861,0.44115035235881805,0.36855583034455774,200,0.8218813121318818,1.0770739316940308,1.2469542503356934,1.051767386496067 +fresh-zh-2,6,7,JS,weak_half,200,0.0004128603759454564,0.0015489699435420334,0.0038255024701356883,0.0019490687905272352,200,0.0007699567766394468,0.0033198274904862046,0.010109059326350688,0.004936716917145531 +fresh-zh-2,6,7,JS,strong_half,200,0.01499104928225279,0.03833772800862789,0.07766694203019142,0.04377788484562188,200,0.014382896572351455,0.03063864354044199,0.05779479779303074,0.033635521156247704 +fresh-zh-2,6,7,JS,random_half,200,0.008502493984997273,0.024180701933801174,0.05359563007950782,0.02908199514669832,200,0.0061460350640118126,0.01569557376205921,0.03703090101480484,0.01932553757505957 +fresh-zh-2,6,7,JS,all_heads,200,0.017577710561454296,0.04498828761279583,0.08648064285516738,0.05094116823049262,200,0.015560419671237469,0.0357669647783041,0.06376001685857773,0.03857223825994879 +fresh-zh-2,6,7,sqrtJS,weak_half,200,0.020317677035927774,0.039356889203190804,0.061850386857986446,0.040630190477240834,200,0.06593088507652284,0.13880279660224915,0.2321288600564003,0.14570703141391278 +fresh-zh-2,6,7,sqrtJS,strong_half,200,0.12243728935718537,0.19579963386058807,0.27868716418743134,0.19998195789754392,200,0.27686722874641423,0.43009863793849945,0.5795172154903412,0.43096388019621373 +fresh-zh-2,6,7,sqrtJS,random_half,200,0.09220896661281586,0.15550142526626587,0.2315070316195488,0.160118302218616,200,0.17432007789611817,0.2806647717952728,0.419491308927536,0.28962112993001937 +fresh-zh-2,6,7,sqrtJS,all_heads,200,0.1325807213783264,0.21210435777902603,0.294072487950325,0.21583859350532295,200,0.34615332186222075,0.5831202566623688,0.7570074021816253,0.5766709113121032 +fresh-zh-2,7,8,Q,weak_half,400,0.11501991301774979,0.20768208801746368,0.40987436771392827,0.2380837624333799,400,0.27755335569381717,0.4664519280195236,0.9416283428668978,0.5448388139903545 +fresh-zh-2,7,8,Q,strong_half,400,0.19946496337652206,0.2939010560512543,0.5158799171447755,0.3386358293518424,400,0.6831974804401398,1.0592145919799805,2.0394862413406374,1.2236409627646208 +fresh-zh-2,7,8,Q,random_half,400,0.18262053281068802,0.3135285973548889,0.5809343338012698,0.3548115189373493,400,0.4343101501464844,0.771829754114151,1.4975882768630988,0.8792067469656467 +fresh-zh-2,7,8,Q,all_heads,400,0.21819276958703995,0.30938318371772766,0.4985089987516404,0.3341772097349167,400,0.9632518887519836,1.5584744811058044,2.9021291494369508,1.7684797801077365 +fresh-zh-2,7,8,K,weak_half,400,0.06171932294964792,0.1890415921807289,0.35289697945117954,0.2069106152560562,400,0.12757752388715746,0.4307650178670883,0.7951205790042878,0.46451918765436856 +fresh-zh-2,7,8,K,strong_half,400,0.07532566636800767,0.2535891681909561,0.4528181284666062,0.2736716274777427,400,0.20490493774414065,0.9827976822853088,1.6328176140785224,1.0146329025086016 +fresh-zh-2,7,8,K,random_half,400,0.09977211505174637,0.3185586482286453,0.5694898962974551,0.343400358739309,400,0.16378118842840195,0.6967650949954987,1.275553667545319,0.7474172781175003 +fresh-zh-2,7,8,K,all_heads,400,0.0930837780237198,0.2764410078525543,0.45368519723415374,0.2865029306989163,400,0.3344550520181656,1.423342227935791,2.408900570869446,1.4791520913504064 +fresh-zh-2,7,8,V,weak_half,400,0.0836452715098858,0.15901128947734833,0.3170878648757935,0.18663387305103243,400,0.18421235531568528,0.3334294557571411,0.587132352590561,0.3728870605304837 +fresh-zh-2,7,8,V,strong_half,400,0.11221926733851434,0.20564604550600052,0.41677319109439853,0.24702762244269252,400,0.4338366597890855,0.9384787976741791,1.571258294582367,0.9727864551916718 +fresh-zh-2,7,8,V,random_half,400,0.1299824059009552,0.2554609328508377,0.5554963529109955,0.31306485299021003,400,0.2827056139707566,0.6388644278049469,1.1172404766082764,0.6712360151484609 +fresh-zh-2,7,8,V,all_heads,400,0.12425580471754075,0.22270255535840988,0.3851474612951279,0.24315173914656044,400,0.6318522036075592,1.3380409479141235,2.1230419635772706,1.3456735160946847 +fresh-zh-2,7,8,QKV,weak_half,400,0.11496670842170716,0.19873535633087158,0.3526554644107819,0.2209978123754263,400,0.266403466463089,0.4254317134618759,0.7772294938564304,0.4886315958201885 +fresh-zh-2,7,8,QKV,strong_half,400,0.1874774217605591,0.2716352641582489,0.4429545253515244,0.304132877252996,400,0.6623718202114105,1.098257303237915,1.6455845355987553,1.1373977293819189 +fresh-zh-2,7,8,QKV,random_half,400,0.19740896821022036,0.32282717525959015,0.5429169952869416,0.35815048607066274,400,0.4427905410528183,0.7476390600204468,1.2850973486900332,0.8130998205393553 +fresh-zh-2,7,8,QKV,all_heads,400,0.20188812464475633,0.28497499227523804,0.41363763809204107,0.3018641635030508,400,0.9557942450046539,1.5293591022491455,2.4166606426239015,1.6260293270647526 +fresh-zh-2,8,9,Q,weak_half,400,0.07672935500741004,0.11229466646909714,0.17892846316099167,0.12136435554362833,400,0.20993089377880098,0.3149445354938507,0.45924491584300997,0.3289245844259858 +fresh-zh-2,8,9,Q,strong_half,400,0.1404616564512253,0.22094734013080597,0.3452083826065064,0.23730728073045612,400,0.4530203700065613,0.6845055818557739,0.9667883217334748,0.7027065766602755 +fresh-zh-2,8,9,Q,random_half,400,0.11377490386366844,0.18117336928844452,0.276766735315323,0.18846340665593744,400,0.3158128082752228,0.4975893795490265,0.7391941249370576,0.5138781623542309 +fresh-zh-2,8,9,Q,all_heads,400,0.16451882272958757,0.25331707298755646,0.39579310119152067,0.2692842585034668,400,0.6823213219642639,1.008996069431305,1.4070010662078858,1.031631160825491 +fresh-zh-2,8,9,K,weak_half,400,0.07525529637932778,0.12215939536690712,0.20642378479242326,0.1333774654008448,400,0.19966924786567689,0.3087341636419296,0.4792025297880173,0.3226432166621089 +fresh-zh-2,8,9,K,strong_half,400,0.14392175823450087,0.21677736192941666,0.31460041999816896,0.22399824809283017,400,0.4595452755689621,0.660209983587265,0.9206157863140106,0.6822025449573994 +fresh-zh-2,8,9,K,random_half,400,0.11997352242469787,0.18345434963703156,0.2731053471565247,0.1924271053262055,400,0.32488200068473816,0.47791096568107605,0.7083921015262604,0.5031368070468306 +fresh-zh-2,8,9,K,all_heads,400,0.16503423005342485,0.25061871111392975,0.349559885263443,0.25300461681559683,400,0.685412472486496,0.9712583720684052,1.3763785123825074,1.004845767095685 +fresh-zh-2,8,9,V,weak_half,400,0.0549673080444336,0.10125114396214485,0.18332651704549802,0.11575645731762052,400,0.14570804536342621,0.2593185156583786,0.431455072760582,0.2783632672391832 +fresh-zh-2,8,9,V,strong_half,400,0.09550895914435387,0.17410146445035934,0.31076946556568147,0.19114395816810428,400,0.3291195809841156,0.5502854585647583,0.972139084339142,0.6159130980446935 +fresh-zh-2,8,9,V,random_half,400,0.0767095796763897,0.14677728712558746,0.26141675710678103,0.1618831360526383,400,0.23577340096235275,0.40759365260601044,0.6878419518470764,0.4447588370367885 +fresh-zh-2,8,9,V,all_heads,400,0.10555323883891106,0.19807353615760803,0.335186716914177,0.21496002453379334,400,0.4854343354701996,0.8241976201534271,1.3908112049102785,0.8942763597518205 +fresh-zh-2,8,9,QKV,weak_half,400,0.0825660578906536,0.11825376749038696,0.18576323390007024,0.1274705027230084,400,0.21873538345098495,0.31146709620952606,0.45126571655273445,0.3250903274863958 +fresh-zh-2,8,9,QKV,strong_half,400,0.15552083402872086,0.2182495892047882,0.30073313117027284,0.22546689085662364,400,0.5053018510341645,0.6926010549068451,0.9190265893936158,0.7052472434937954 +fresh-zh-2,8,9,QKV,random_half,400,0.12426191419363022,0.17980801314115524,0.25862217843532564,0.18774775179103018,400,0.3547805666923523,0.4962863177061081,0.6934911608695984,0.513624276816845 +fresh-zh-2,8,9,QKV,all_heads,400,0.172349950671196,0.24899354577064514,0.34364267587661745,0.25432584673166275,400,0.7290561497211456,1.0042466521263123,1.368918514251709,1.0303375677764415 +fresh-zh-2,9,10,Q,weak_half,400,0.060545417666435244,0.10804775729775429,0.17530018985271456,0.11492105473764241,400,0.16587185263633727,0.27295735478401184,0.4397276490926743,0.29217840200290085 +fresh-zh-2,9,10,Q,strong_half,400,0.1323890805244446,0.22104863822460175,0.39098266661167147,0.24610418250784277,400,0.40542857348918915,0.6548906564712524,1.0483936429023744,0.6989319017529487 +fresh-zh-2,9,10,Q,random_half,400,0.10141102969646454,0.1676434725522995,0.2931577503681183,0.1861703429557383,400,0.2809086859226227,0.4608035832643509,0.7649609327316285,0.4999809089675546 +fresh-zh-2,9,10,Q,all_heads,400,0.14739016741514208,0.2497135028243065,0.4095666199922563,0.2675049745477736,400,0.5940493643283844,0.9190406203269958,1.46542706489563,0.9911103043705225 +fresh-zh-2,9,10,K,weak_half,400,0.07361450716853142,0.11231904476881027,0.18114440590143205,0.12058835410512984,400,0.18395409435033802,0.26890386641025543,0.3893064618110657,0.2803748985566199 +fresh-zh-2,9,10,K,strong_half,400,0.1566017895936966,0.2219035029411316,0.3203972488641739,0.23414769142866135,400,0.4525820195674896,0.6202647984027863,0.8345050096511841,0.6311703454703093 +fresh-zh-2,9,10,K,random_half,400,0.12236895710229874,0.18497807532548904,0.2729706645011902,0.19150492556393148,400,0.30276575684547424,0.44094571471214294,0.6152169346809387,0.45466623999178407 +fresh-zh-2,9,10,K,all_heads,400,0.1725284591317177,0.2578907459974289,0.3678524613380433,0.26430909648537637,400,0.6508710503578187,0.8990921676158905,1.2014237642288208,0.9115452484041452 +fresh-zh-2,9,10,V,weak_half,400,0.05010535679757595,0.0895957462489605,0.16139430552721024,0.1005913665983826,400,0.1365443304181099,0.2147623598575592,0.3854637205600739,0.2387964224629104 +fresh-zh-2,9,10,V,strong_half,400,0.10140954777598381,0.1644650250673294,0.300866374373436,0.18747598493471743,400,0.31954190135002136,0.472663089632988,0.7787706017494203,0.516890792362392 +fresh-zh-2,9,10,V,random_half,400,0.08075519800186158,0.13314945995807648,0.24440934956073765,0.15218206870369613,400,0.22031891942024232,0.33908718824386597,0.596290534734726,0.37568548537790775 +fresh-zh-2,9,10,V,all_heads,400,0.11862896978855135,0.1900293156504631,0.33564236760139465,0.2116646590642631,400,0.4587136507034302,0.6876073479652405,1.1429073572158814,0.7556872136890889 +fresh-zh-2,9,10,QKV,weak_half,400,0.07327372878789902,0.11009984463453293,0.16982217729091648,0.11635067685507237,400,0.18595506101846696,0.27199462056159973,0.39652351140975955,0.2834281402081251 +fresh-zh-2,9,10,QKV,strong_half,400,0.1572491094470024,0.22345035523176193,0.31761303544044495,0.2317875382862985,400,0.46439705789089203,0.6388565897941589,0.8704236090183258,0.6523337741941213 +fresh-zh-2,9,10,QKV,random_half,400,0.1219222940504551,0.17968101799488068,0.24993446022272112,0.18396044010296464,400,0.3125755786895752,0.45801016688346863,0.6356663823127746,0.46895845342427495 +fresh-zh-2,9,10,QKV,all_heads,400,0.1736375004053116,0.2547629326581955,0.3381987035274506,0.2575250355154276,400,0.6504404902458191,0.9181888699531555,1.2623573541641235,0.9357619156688451 +fresh-zh-2,10,11,Q,weak_half,200,0.05809122286736965,0.09526727721095085,0.1373703196644783,0.09648039560765027,200,0.1565330594778061,0.26065221428871155,0.3584189087152481,0.25807913109660147 +fresh-zh-2,10,11,Q,strong_half,200,0.17240832149982452,0.22608089447021484,0.29141342341899873,0.2331640786677599,200,0.427206227183342,0.5938786566257477,0.7376748919487,0.5916289655864239 +fresh-zh-2,10,11,Q,random_half,200,0.11979197710752487,0.17656272649765015,0.22600523829460145,0.17498455591499806,200,0.299173566699028,0.4419882148504257,0.5570366680622101,0.42884018607437613 +fresh-zh-2,10,11,Q,all_heads,200,0.1948069676756859,0.2522839605808258,0.33278900384902954,0.2584721230715513,200,0.618204265832901,0.8430793881416321,1.0857965469360351,0.849708099514246 +fresh-zh-2,10,11,K,weak_half,200,0.048389685526490214,0.07653293013572693,0.10732165724039078,0.07761635908856988,200,0.12198185920715332,0.19720731675624847,0.26444498300552366,0.1970194958150387 +fresh-zh-2,10,11,K,strong_half,200,0.18663746416568755,0.2731491029262543,0.3194005489349365,0.26099538668990135,200,0.4814393639564514,0.6247996687889099,0.7281221449375153,0.608902145922184 +fresh-zh-2,10,11,K,random_half,200,0.11615855172276497,0.17063888907432556,0.2703566700220108,0.18257811184972525,200,0.2725476533174515,0.3942382484674454,0.5384599208831787,0.39989000380039214 +fresh-zh-2,10,11,K,all_heads,200,0.1938636064529419,0.2882729470729828,0.33853906095027925,0.2808468729257584,200,0.6179399847984314,0.8210435211658478,0.9326142728328705,0.8059216451644897 +fresh-zh-2,10,11,V,weak_half,200,0.07253863662481308,0.11250614374876022,0.15310460180044175,0.11320402072742582,200,0.1873881220817566,0.2916935086250305,0.3835133910179138,0.2959113029390574 +fresh-zh-2,10,11,V,strong_half,200,0.20068129152059555,0.2452135980129242,0.30775807499885555,0.2530645336955786,200,0.5374365985393524,0.6624546051025391,0.7968273639678953,0.6653106039762497 +fresh-zh-2,10,11,V,random_half,200,0.14239993691444397,0.1840181052684784,0.25388412177562714,0.19115967396646738,200,0.34835387468338014,0.47605299949645996,0.601543116569519,0.4821298637241125 +fresh-zh-2,10,11,V,all_heads,200,0.21512978076934816,0.27266331017017365,0.3435991615056991,0.27941328793764114,200,0.74908247590065,0.9892854392528534,1.1872108697891235,0.9612218999862671 +fresh-zh-2,10,11,QKV,weak_half,200,0.06158177256584168,0.09912759810686111,0.13022897094488145,0.09738719949498773,200,0.16031771302223205,0.2665250301361084,0.33654427230358125,0.2561560580506921 +fresh-zh-2,10,11,QKV,strong_half,200,0.21092249006032943,0.24656762182712555,0.3026114284992218,0.25125792682170867,200,0.5289937257766724,0.6344618797302246,0.7748373687267304,0.6410838364064694 +fresh-zh-2,10,11,QKV,random_half,200,0.1345460221171379,0.1853308603167534,0.23595131933689117,0.18537638511508703,200,0.3279202163219452,0.45175987482070923,0.5574859499931335,0.4490484444797039 +fresh-zh-2,10,11,QKV,all_heads,200,0.2197829142212868,0.2678325027227402,0.33001674115657803,0.27486071430146697,200,0.6919123709201813,0.9175626039505005,1.079559338092804,0.8972398939728737 +fresh-zh-2,10,11,JS,weak_half,200,0.0006725717568770051,0.0025995317846536636,0.007425001868978142,0.0033351748136192325,200,0.0011692118016071618,0.0032229265198111534,0.007380979601293802,0.003822854450554587 +fresh-zh-2,10,11,JS,strong_half,200,0.01148390220478177,0.02528925985097885,0.06133109517395495,0.03654496079543605,200,0.008839023951441049,0.020151750184595585,0.04590754956007003,0.028978726809145882 +fresh-zh-2,10,11,JS,random_half,200,0.005910992389544845,0.0160494614392519,0.037735163792967784,0.022933436106541194,200,0.004426252888515592,0.010926312766969204,0.024178072810173035,0.017886144001386127 +fresh-zh-2,10,11,JS,all_heads,200,0.015325034223496915,0.031197580508887768,0.06463769376277924,0.04094371306709945,200,0.011142155900597573,0.02416140679270029,0.05232254453003405,0.032801581148523835 +fresh-zh-2,10,11,sqrtJS,weak_half,200,0.02593398727476597,0.05098559893667698,0.08616753965616225,0.05315380671061575,200,0.08764518350362778,0.14907324314117432,0.22522300779819487,0.15235219521448015 +fresh-zh-2,10,11,sqrtJS,strong_half,200,0.1071628674864769,0.15902575105428696,0.24764539450407025,0.17566346250474452,200,0.23531716912984849,0.3615851402282715,0.5139065504074096,0.3682800804451108 +fresh-zh-2,10,11,sqrtJS,random_half,200,0.07688297629356385,0.12668617069721222,0.19424909651279446,0.1351506732031703,200,0.1681055963039398,0.24901407212018967,0.3730812132358551,0.26297642232850194 +fresh-zh-2,10,11,sqrtJS,all_heads,200,0.12379417940974236,0.1766282021999359,0.25423933267593385,0.1890742003545165,200,0.3440644860267639,0.5116505026817322,0.7025925397872924,0.5206322740763426 +fresh-zh-2,11,12,Q,weak_half,400,0.11733006834983825,0.2020385041832924,0.35670980513095857,0.22236370101571082,400,0.24258084893226625,0.34465840458869934,0.543546211719513,0.3697677306085825 +fresh-zh-2,11,12,Q,strong_half,400,0.14845186620950698,0.21940569579601288,0.35043988823890687,0.23546066496521234,400,0.45972551703453063,0.708428293466568,1.1401207327842713,0.7551849377900361 +fresh-zh-2,11,12,Q,random_half,400,0.11865318417549134,0.2127816379070282,0.3853861302137375,0.23908725893124938,400,0.3201279163360596,0.5117341876029968,0.8994354665279388,0.5627735051885248 +fresh-zh-2,11,12,Q,all_heads,400,0.12425258904695512,0.1905544474720955,0.28566747903823864,0.19847672471776603,400,0.7174283981323243,1.0416491031646729,1.6478107094764711,1.1249526748806238 +fresh-zh-2,11,12,K,weak_half,400,0.10207040533423424,0.18312282115221024,0.3340327262878418,0.20212115602567793,400,0.17535753697156906,0.2828668802976608,0.4704534292221069,0.30775592202320695 +fresh-zh-2,11,12,K,strong_half,400,0.10198884606361389,0.17936693876981735,0.28109371066093447,0.1900352643057704,400,0.3209595799446106,0.5568952858448029,0.9768767476081848,0.6016723706200718 +fresh-zh-2,11,12,K,random_half,400,0.10724460259079935,0.19250735640525818,0.4039032459259034,0.23086604241281747,400,0.23453251719474794,0.42394983768463135,0.752253317832947,0.46019582338631154 +fresh-zh-2,11,12,K,all_heads,400,0.10352343246340752,0.17076624929904938,0.27928519845008865,0.18467908412218093,400,0.4974952101707459,0.8568938970565796,1.4117163658142091,0.9094282945245504 +fresh-zh-2,11,12,V,weak_half,400,0.07290947139263153,0.14211178570985794,0.2694735020399094,0.16058510250411928,400,0.15088752657175064,0.2439786046743393,0.4074105501174927,0.26545343806967137 +fresh-zh-2,11,12,V,strong_half,400,0.0806106261909008,0.13971736282110214,0.2555760562419892,0.15896635301411152,400,0.24169393628835678,0.47515323758125305,0.8610004544258122,0.5290280875191092 +fresh-zh-2,11,12,V,random_half,400,0.07585591450333595,0.16047478467226028,0.30655547380447395,0.18605775559321047,400,0.1876835495233536,0.3586433529853821,0.6795750677585602,0.3993683055602014 +fresh-zh-2,11,12,V,all_heads,400,0.08060992285609245,0.14583168178796768,0.24802519828081135,0.15804083393886686,400,0.40627951323986056,0.7250699996948242,1.253320598602295,0.7944815232604742 +fresh-zh-2,11,12,QKV,weak_half,400,0.12089074924588203,0.19041018187999725,0.3131822109222412,0.20355342708528043,400,0.22189488708972932,0.3087073117494583,0.4663683146238328,0.3286345063894987 +fresh-zh-2,11,12,QKV,strong_half,400,0.13642823994159697,0.1965373381972313,0.2822093576192856,0.20374872624874116,400,0.40719278752803806,0.630268394947052,0.9495732843875885,0.6624377898871898 +fresh-zh-2,11,12,QKV,random_half,400,0.12424726262688637,0.21461418271064758,0.3536228984594345,0.22945917885750533,400,0.3021746277809143,0.45107603073120117,0.7554274857044221,0.49821656573563816 +fresh-zh-2,11,12,QKV,all_heads,400,0.1260660246014595,0.1817675232887268,0.2506496548652649,0.18695072133094073,400,0.6409212946891785,0.9385963082313538,1.391756021976471,0.9910722962021827 +fresh-zh-2,12,13,Q,weak_half,400,0.04960459545254708,0.07762378081679344,0.11973500996828079,0.08252789679449052,400,0.1381758362054825,0.21506193280220032,0.31042827665805817,0.22217238960787655 +fresh-zh-2,12,13,Q,strong_half,400,0.10421309620141983,0.1715901419520378,0.3048519104719163,0.19051309660077095,400,0.32740226984024046,0.5173593759536743,0.7945390760898594,0.5428747995570302 +fresh-zh-2,12,13,Q,random_half,400,0.0752455398440361,0.13010620325803757,0.22196723371744156,0.14327647837810217,400,0.22489532828330994,0.35626721382141113,0.5740774989128115,0.3868368522822857 +fresh-zh-2,12,13,Q,all_heads,400,0.1107637569308281,0.18118860572576523,0.3078986644744874,0.20230572551488876,400,0.46481805443763735,0.7350729703903198,1.1088959813117982,0.7650471866875886 +fresh-zh-2,12,13,K,weak_half,400,0.0501171175390482,0.08239372447133064,0.12552271038293844,0.08610711858607828,400,0.12639287263154983,0.19831881672143936,0.29752256870269783,0.20515652680769564 +fresh-zh-2,12,13,K,strong_half,400,0.09459462314844132,0.14579559117555618,0.2152704641222954,0.15354661613702775,400,0.29482573866844175,0.4232279360294342,0.5921652436256409,0.43500331826508043 +fresh-zh-2,12,13,K,random_half,400,0.07906277626752854,0.1232818253338337,0.19158277809619906,0.13013536700047554,400,0.20604025572538376,0.306414470076561,0.4542174696922303,0.32120740212500093 +fresh-zh-2,12,13,K,all_heads,400,0.10565693229436875,0.1641126349568367,0.24348812848329546,0.17089538416825234,400,0.43644048273563385,0.6150440573692322,0.885837858915329,0.6401598461717367 +fresh-zh-2,12,13,V,weak_half,400,0.03661304228007793,0.06362244114279747,0.10754167288541794,0.06990037212148309,400,0.10231047123670578,0.15794268250465393,0.25045901238918306,0.1697707894258201 +fresh-zh-2,12,13,V,strong_half,400,0.06616248339414596,0.11187358945608139,0.1953046903014183,0.12525101948529482,400,0.2437355786561966,0.35896436870098114,0.5324044704437256,0.37770062319934367 +fresh-zh-2,12,13,V,random_half,400,0.054050813987851144,0.0978706032037735,0.16992778033018113,0.1061772226402536,400,0.1709301471710205,0.2604460120201111,0.3996974408626557,0.2771344132907689 +fresh-zh-2,12,13,V,all_heads,400,0.0768166534602642,0.12282830476760864,0.208562283217907,0.13613640232011676,400,0.36192336976528167,0.517450362443924,0.762963879108429,0.547471412718296 +fresh-zh-2,12,13,QKV,weak_half,400,0.054570150747895244,0.07972760498523712,0.11325867027044297,0.08206341951154172,400,0.13953581005334853,0.20597374439239502,0.2818590492010117,0.20780314395204186 +fresh-zh-2,12,13,QKV,strong_half,400,0.11103901565074921,0.15861456841230392,0.22104950100183493,0.16530200285837054,400,0.35915649533271793,0.47251249849796295,0.6258534491062164,0.48323188569396736 +fresh-zh-2,12,13,QKV,random_half,400,0.0892927698791027,0.12828603386878967,0.18128076493740083,0.1329892193712294,400,0.23933622241020203,0.3358944058418274,0.48042560517787936,0.34910066101700066 +fresh-zh-2,12,13,QKV,all_heads,400,0.1216454602777958,0.17226263135671616,0.2381416827440262,0.1784146299213171,400,0.508368867635727,0.6778793334960938,0.8920386791229249,0.691035027205944 +fresh-zh-2,13,14,Q,weak_half,400,0.034263442084193234,0.061843086034059525,0.11240421384572986,0.06782323581399397,400,0.0950643926858902,0.16934867948293686,0.27975376844406136,0.17758740260265768 +fresh-zh-2,13,14,Q,strong_half,400,0.10446888655424119,0.16418351233005524,0.273970827460289,0.1780627867951989,400,0.2922616064548492,0.47325460612773895,0.7547839045524598,0.5002639067545533 +fresh-zh-2,13,14,Q,random_half,400,0.06292319446802139,0.12151537835597992,0.20902744680643084,0.1300126162637025,400,0.17679038643836978,0.3253759741783142,0.5466394662857056,0.3435565176140517 +fresh-zh-2,13,14,Q,all_heads,400,0.10570311471819878,0.17729689925909042,0.296221587061882,0.19067085539456458,400,0.40826204419136053,0.6566768884658813,1.0182255387306214,0.6778513112664223 +fresh-zh-2,13,14,K,weak_half,400,0.03546931780874729,0.07140714302659035,0.11390213146805764,0.07452516647754237,400,0.09593984559178353,0.17257798463106155,0.28700693547725675,0.18615194973535837 +fresh-zh-2,13,14,K,strong_half,400,0.10039940103888512,0.19237544387578964,0.281455796957016,0.19326661992818117,400,0.29614495635032656,0.5029185116291046,0.7716325640678406,0.521335321571678 +fresh-zh-2,13,14,K,random_half,400,0.07979629412293436,0.1527278870344162,0.26169114708900454,0.16290841813664883,400,0.1939662888646126,0.3432449400424957,0.5370608627796175,0.35928574349731207 +fresh-zh-2,13,14,K,all_heads,400,0.11056661754846574,0.20081477612257004,0.28768795132637026,0.20230605257675052,400,0.40258807241916655,0.688849151134491,1.0459978461265564,0.707487269565463 +fresh-zh-2,13,14,V,weak_half,400,0.02696644812822342,0.05167106352746487,0.1258958220481873,0.06375111980829389,400,0.07627935335040092,0.13779661059379578,0.2595121681690218,0.1589990052767098 +fresh-zh-2,13,14,V,strong_half,400,0.0743052065372467,0.1363629847764969,0.3007320672273636,0.16623309971764683,400,0.2255341649055481,0.37379077076911926,0.7039154410362244,0.4337347562611103 +fresh-zh-2,13,14,V,random_half,400,0.05532718189060688,0.10138217732310295,0.24353006631135946,0.13248815638944506,400,0.14722223728895187,0.24578191339969635,0.5056994915008546,0.29764602789655326 +fresh-zh-2,13,14,V,all_heads,400,0.0845200575888157,0.1500200629234314,0.3258771896362305,0.17817630601115525,400,0.3138166666030884,0.511152595281601,0.9712536990642548,0.5927337601780891 +fresh-zh-2,13,14,QKV,weak_half,400,0.038317539170384406,0.0679476447403431,0.11137442588806153,0.07183737435378135,400,0.10186097323894501,0.17649385333061218,0.2765123873949051,0.1832050807401538 +fresh-zh-2,13,14,QKV,strong_half,400,0.10863361805677414,0.18030710518360138,0.2676051646471024,0.18669939960353077,400,0.31612505912780764,0.4904506206512451,0.7299620449543001,0.515068173520267 +fresh-zh-2,13,14,QKV,random_half,400,0.08271794319152832,0.13795767724514008,0.23157390356063842,0.14965369687415658,400,0.20218586325645446,0.3404891937971115,0.513986110687256,0.3531366855464876 +fresh-zh-2,13,14,QKV,all_heads,400,0.11896158829331399,0.19179635494947433,0.28278276026248933,0.19807124366983772,400,0.43320168256759645,0.6724704504013062,0.99070662856102,0.6982732547074556 +fresh-zh-2,14,15,Q,weak_half,200,0.06271082982420921,0.09818928316235542,0.1374141991138458,0.10025042451918126,200,0.15960971862077714,0.2548031806945801,0.3649199217557907,0.260791554749012 +fresh-zh-2,14,15,Q,strong_half,200,0.20739603489637376,0.2604239135980606,0.32789650857448577,0.26307900428771974,200,0.5208862066268921,0.6749774217605591,0.8492313086986542,0.6832216593623162 +fresh-zh-2,14,15,Q,random_half,200,0.1332329750061035,0.1858796402812004,0.2619890540838241,0.19161722406744958,200,0.32004196345806124,0.452532097697258,0.6300119638442992,0.46885129913687706 +fresh-zh-2,14,15,Q,all_heads,200,0.21011206656694412,0.2695990204811096,0.3465628683567047,0.27365672059357166,200,0.7097750127315521,0.947596549987793,1.196447491645813,0.9440132120251655 +fresh-zh-2,14,15,K,weak_half,200,0.035029929131269455,0.051041170954704285,0.06720122247934342,0.0502888788562268,200,0.08443593755364419,0.12241147458553314,0.16805352866649628,0.12211694825440646 +fresh-zh-2,14,15,K,strong_half,200,0.15225592851638795,0.20143530517816544,0.3604703992605209,0.24235865145921706,200,0.3512906312942505,0.4844953417778015,0.7651932656764984,0.5361148172616959 +fresh-zh-2,14,15,K,random_half,200,0.0760440967977047,0.1316063404083252,0.3042240738868712,0.15653631461784243,200,0.1872629314661026,0.27686960995197296,0.5337638974189758,0.3137336556240916 +fresh-zh-2,14,15,K,all_heads,200,0.161360639333725,0.20902752131223679,0.3632954835891723,0.24856786370277406,200,0.46462448537349704,0.6070328950881958,0.9176524996757507,0.6582317668199539 +fresh-zh-2,14,15,V,weak_half,200,0.036642830818891525,0.049899060279130936,0.06944888383150101,0.051860287226736544,200,0.09075915813446045,0.1187230795621872,0.1662780299782753,0.12562856417149304 +fresh-zh-2,14,15,V,strong_half,200,0.13302152007818221,0.1983097344636917,0.28941326141357415,0.20979572389274836,200,0.33187415897846223,0.4504379630088806,0.612033677101135,0.4686401551961899 +fresh-zh-2,14,15,V,random_half,200,0.07412829026579858,0.13107771426439285,0.22268537282943726,0.13841151064261795,200,0.17025825381278992,0.26764683425426483,0.4214413821697235,0.2846737751364708 +fresh-zh-2,14,15,V,all_heads,200,0.15250787287950518,0.2052433043718338,0.2950038433074951,0.21953585505485534,200,0.4360881626605988,0.5877294838428497,0.7734893798828125,0.5942687273025513 +fresh-zh-2,14,15,QKV,weak_half,200,0.0504409421235323,0.07073170319199562,0.0928923636674881,0.07201128439977765,200,0.12470831274986267,0.18187545239925385,0.25031186938285827,0.18586763564497233 +fresh-zh-2,14,15,QKV,strong_half,200,0.20300718396902084,0.24744059145450592,0.2872049570083618,0.2471238097548485,200,0.5163794696331024,0.6219441890716553,0.7110221982002258,0.6174947316944599 +fresh-zh-2,14,15,QKV,random_half,200,0.11219120025634766,0.16850487887859344,0.23497135937213898,0.1707809848897159,200,0.2587660074234009,0.3867143988609314,0.5090081453323364,0.3900500972568989 +fresh-zh-2,14,15,QKV,all_heads,200,0.21140896826982497,0.2555245906114578,0.297117206454277,0.2554413276165724,200,0.6588486015796662,0.7985384464263916,0.9420318186283112,0.8033623656630516 +fresh-zh-2,14,15,JS,weak_half,200,0.000712060375371948,0.002641250961460173,0.00798478852957487,0.0038523611226446517,200,0.0009664817596785724,0.002923916676081717,0.005973426904529332,0.0033972052281023932 +fresh-zh-2,14,15,JS,strong_half,200,0.01074603106826544,0.024172890931367874,0.06416026055812835,0.031952822455205025,200,0.012516664620488883,0.02777958009392023,0.06388873755931854,0.03430402033380233 +fresh-zh-2,14,15,JS,random_half,200,0.004988785553723574,0.013722216244786978,0.03948715589940548,0.01952113633684348,200,0.005256434995681048,0.011848065070807934,0.0360194019973278,0.017890948010317514 +fresh-zh-2,14,15,JS,all_heads,200,0.010867191012948751,0.02540552243590355,0.06286080330610275,0.032626416739076375,200,0.015120767802000046,0.031155623495578766,0.06729763150215148,0.037701225415803494 +fresh-zh-2,14,15,sqrtJS,weak_half,200,0.026683922857046127,0.051392899826169014,0.08935703858733177,0.0556263252813369,200,0.0785523608326912,0.13475202769041061,0.19720925390720367,0.1380664858594537 +fresh-zh-2,14,15,sqrtJS,strong_half,200,0.1036574922502041,0.15547632426023483,0.25329810082912446,0.1678675713762641,200,0.28147317171096803,0.40651051700115204,0.571086174249649,0.418818478807807 +fresh-zh-2,14,15,sqrtJS,random_half,200,0.07063130363821983,0.11714175343513489,0.19871238023042678,0.12820306746289134,200,0.16536769568920134,0.26090459525585175,0.4012069284915924,0.27285549800843 +fresh-zh-2,14,15,sqrtJS,all_heads,200,0.10424575358629226,0.1593756079673767,0.2507202237844467,0.17021997287869453,200,0.4032789617776871,0.536505401134491,0.7292719304561615,0.5568849652260542 +fresh-zh-2,15,16,Q,weak_half,400,0.05296416990458966,0.1097058579325676,0.20981872379779826,0.12176609385292977,400,0.09573695659637452,0.19757343083620071,0.33399322628974915,0.20655532903037965 +fresh-zh-2,15,16,Q,strong_half,400,0.07280706465244294,0.1799856796860695,0.35194335877895366,0.20504566206596792,400,0.15536409169435503,0.3523363322019577,0.713522094488144,0.4003810223005712 +fresh-zh-2,15,16,Q,random_half,400,0.058560377731919286,0.1475047618150711,0.28840276002883913,0.1628877080185339,400,0.11904392018914223,0.27504539489746094,0.5124441504478454,0.2966858225688338 +fresh-zh-2,15,16,Q,all_heads,400,0.11578345969319347,0.2294987589120865,0.4293626874685288,0.2573411089926958,400,0.2486498832702637,0.5586209893226624,1.04012633562088,0.6069363518059254 +fresh-zh-2,15,16,K,weak_half,400,0.06731320694088937,0.12379990890622139,0.22896444648504266,0.13868783951271324,400,0.1278907224535942,0.1983300894498825,0.3210069835186005,0.21365824959240853 +fresh-zh-2,15,16,K,strong_half,400,0.10169426426291465,0.1691870465874672,0.28708243072032935,0.18275384192354976,400,0.2085164651274681,0.3523632735013962,0.6660577952861787,0.3972750799544156 +fresh-zh-2,15,16,K,random_half,400,0.08386149182915688,0.1546521857380867,0.2826735109090806,0.17821838243864477,400,0.16606896221637726,0.27462275326251984,0.47131496071815515,0.2997093195281923 +fresh-zh-2,15,16,K,all_heads,400,0.14613330215215684,0.23343699425458908,0.3541532337665559,0.24448163107037543,400,0.3425259649753571,0.5605677366256714,0.9642517924308778,0.6109333278611302 +fresh-zh-2,15,16,V,weak_half,400,0.06655594855546952,0.1148284561932087,0.191215455532074,0.12341099446173757,400,0.12295637577772141,0.1802930235862732,0.2718257755041123,0.19074043171480298 +fresh-zh-2,15,16,V,strong_half,400,0.10166022330522537,0.1639944389462471,0.26632468998432174,0.1762286716233939,400,0.2128176808357239,0.31863120198249817,0.4956288427114492,0.34651110403239727 +fresh-zh-2,15,16,V,random_half,400,0.08618020489811898,0.13986745476722717,0.25576616823673265,0.16139126570895315,400,0.15961675345897675,0.24524610489606857,0.3860152125358582,0.2656296550668776 +fresh-zh-2,15,16,V,all_heads,400,0.1633816361427307,0.22085733711719513,0.3414018601179123,0.23958097053691746,400,0.3388347923755646,0.49067915976047516,0.7781152904033661,0.5372515359893442 +fresh-zh-2,15,16,QKV,weak_half,400,0.07847988680005073,0.12092074006795883,0.20730030238628389,0.1322494880016893,400,0.142325359582901,0.20245439559221268,0.2992523729801178,0.21138129249215126 +fresh-zh-2,15,16,QKV,strong_half,400,0.11575153470039369,0.18263930827379227,0.29984006881713865,0.19604245956987143,400,0.24242400228977204,0.3718942701816559,0.6185309827327728,0.40160632636398075 +fresh-zh-2,15,16,QKV,random_half,400,0.09399177730083466,0.1591888815164566,0.267033988237381,0.17489325745031237,400,0.18424578607082367,0.2768609821796417,0.44382110834121713,0.3002216679789126 +fresh-zh-2,15,16,QKV,all_heads,400,0.1670123815536499,0.24380531907081604,0.36500730216503147,0.25593341447412965,400,0.38741981983184814,0.5657934546470642,0.9107420206069949,0.6129876197874546 +fresh-zh-2,16,17,Q,weak_half,400,0.009935837145894766,0.03415417857468128,0.07075487151741981,0.03800633926992305,400,0.02709893826395273,0.09796687215566635,0.19140416681766514,0.10397233369294553 +fresh-zh-2,16,17,Q,strong_half,400,0.024711767956614495,0.08420652896165848,0.1741850018501282,0.09211719665210694,400,0.06669742912054062,0.2312285304069519,0.4375150859355929,0.24835383663885297 +fresh-zh-2,16,17,Q,random_half,400,0.017365497909486292,0.06281135976314545,0.12709931731224064,0.06787037449539639,400,0.04570143297314644,0.15935274213552475,0.3127641975879669,0.1747891979943961 +fresh-zh-2,16,17,Q,all_heads,400,0.026276730559766294,0.08893999457359314,0.18549943119287496,0.09945361200720072,400,0.09445429518818856,0.3330305367708206,0.6300873994827271,0.3523261696472764 +fresh-zh-2,16,17,K,weak_half,400,0.03447007648646832,0.051822127774357796,0.0867315649986267,0.05574236646760255,400,0.09934577569365502,0.13400931656360626,0.19979822337627412,0.14302446575835348 +fresh-zh-2,16,17,K,strong_half,400,0.09027547240257264,0.12412077933549881,0.1731634944677353,0.12878162619657815,400,0.24980487376451493,0.33491386473178864,0.46032795906066903,0.34580265283584594 +fresh-zh-2,16,17,K,random_half,400,0.062100964784622195,0.0910503976047039,0.13398102670907974,0.09575353187508881,400,0.16836116313934327,0.2290978878736496,0.3244350254535675,0.24051611121743918 +fresh-zh-2,16,17,K,all_heads,400,0.09513578563928604,0.13150649517774582,0.19060864448547363,0.13824683114886283,400,0.3555954545736313,0.46911515295505524,0.6366063237190247,0.48882712043821813 +fresh-zh-2,16,17,V,weak_half,400,0.02432969715446234,0.04414794035255909,0.07485520616173745,0.04800163654377684,400,0.06959629356861115,0.11509216204285622,0.17888563722372056,0.12127980278804898 +fresh-zh-2,16,17,V,strong_half,400,0.08027007654309273,0.11979211121797562,0.1776773124933243,0.12589444532990457,400,0.2194700062274933,0.3206135779619217,0.44321041703224184,0.3294463151320815 +fresh-zh-2,16,17,V,random_half,400,0.05289119817316532,0.08080115541815758,0.1341905638575554,0.08850929488893598,400,0.14214293360710145,0.2160685658454895,0.3284931659698486,0.22326322436332702 +fresh-zh-2,16,17,V,all_heads,400,0.0874842867255211,0.13068774342536926,0.19363214671611786,0.13698556892573832,400,0.30306057929992675,0.4352279603481293,0.613510024547577,0.4507261177897453 +fresh-zh-2,16,17,QKV,weak_half,400,0.032469556853175166,0.047202564775943756,0.07056713998317721,0.050171766253188255,400,0.08848373517394066,0.12584398686885834,0.17646596729755404,0.13091058439575134 +fresh-zh-2,16,17,QKV,strong_half,400,0.08850800096988679,0.11930758133530617,0.15719040334224701,0.12181321386247873,400,0.24187853038311005,0.32637670636177063,0.41468892693519593,0.3295865111425519 +fresh-zh-2,16,17,QKV,random_half,400,0.0596903320401907,0.08615831658244133,0.12263640388846397,0.08896983059123159,400,0.16256119012832643,0.22000296413898468,0.3043559938669205,0.22737730922177435 +fresh-zh-2,16,17,QKV,all_heads,400,0.09345708265900612,0.12850981205701828,0.17520596086978912,0.13192057996988296,400,0.3363911211490631,0.45525990426540375,0.5883902549743653,0.46049709506332875 +fresh-zh-2,17,18,Q,weak_half,400,0.011101483181118966,0.0323680080473423,0.057603910937905324,0.033827579806093126,400,0.031985342130064966,0.0876847617328167,0.15673617571592333,0.09143375111743808 +fresh-zh-2,17,18,Q,strong_half,400,0.0385629877448082,0.10841600596904755,0.18900175839662553,0.1100821303576231,400,0.08775615692138672,0.27139487862586975,0.4259723544120789,0.2626310554984957 +fresh-zh-2,17,18,Q,random_half,400,0.023962982557713987,0.06889569386839867,0.14009916782379153,0.07679417449748144,400,0.061404936015605927,0.17443960905075073,0.2982084572315216,0.17818841366562993 +fresh-zh-2,17,18,Q,all_heads,400,0.04057067260146141,0.11275390163064003,0.19443033486604694,0.11559884711634368,400,0.12266913503408432,0.36147016286849976,0.5818089663982392,0.3540648077335209 +fresh-zh-2,17,18,K,weak_half,400,0.016734050400555135,0.032804613932967186,0.05211637504398823,0.03370113857439719,400,0.043253502994775775,0.0873953104019165,0.1269218221306801,0.08656848683021963 +fresh-zh-2,17,18,K,strong_half,400,0.058700710907578474,0.1025642417371273,0.15583149790763856,0.10609589752741158,400,0.14617993533611298,0.2391187995672226,0.34780321419239046,0.24322532111778855 +fresh-zh-2,17,18,K,random_half,400,0.04161272682249546,0.06676663085818291,0.11925722509622574,0.07517479854403064,400,0.09357281252741814,0.15649239718914032,0.2538058370351791,0.16596424202434717 +fresh-zh-2,17,18,K,all_heads,400,0.06015550643205643,0.10847470164299011,0.16301802843809132,0.11092325892299414,400,0.1949007257819176,0.3274110108613968,0.4664688229560852,0.32979380747303366 +fresh-zh-2,17,18,V,weak_half,400,0.019874605536460876,0.03137070685625076,0.04852171577513219,0.03309973497875035,400,0.05361320488154888,0.08563925698399544,0.12278700545430186,0.08706184229813516 +fresh-zh-2,17,18,V,strong_half,400,0.10827316716313362,0.1447114795446396,0.18504086136817932,0.14562460164539515,400,0.23912318795919418,0.31155066192150116,0.40475570559501656,0.31770736826583745 +fresh-zh-2,17,18,V,random_half,400,0.04249230325222016,0.1011737808585167,0.15419833362102509,0.09843956577125937,400,0.10325845628976822,0.20564565807580948,0.3110764265060425,0.20660887354053556 +fresh-zh-2,17,18,V,all_heads,400,0.11154847592115402,0.14937632530927658,0.18985365480184557,0.14961126811802386,400,0.29345017671585083,0.3991250991821289,0.5174729585647584,0.4047692118212581 +fresh-zh-2,17,18,QKV,weak_half,400,0.022748670354485512,0.034580888226628304,0.04894055239856243,0.03529463768703863,400,0.06198169961571694,0.09218157455325127,0.12754254043102264,0.09314262809231877 +fresh-zh-2,17,18,QKV,strong_half,400,0.09590465575456619,0.12382733821868896,0.16155840605497362,0.12690051000565292,400,0.2223674699664116,0.28712907433509827,0.3669418722391129,0.2926468723267317 +fresh-zh-2,17,18,QKV,random_half,400,0.0485966756939888,0.08914709463715553,0.13051219880580903,0.08856487947516144,400,0.1207292065024376,0.19164200127124786,0.27366073727607726,0.1954509746003896 +fresh-zh-2,17,18,QKV,all_heads,400,0.0981829434633255,0.1303626075387001,0.1655799955129624,0.13169100267812608,400,0.2939887970685959,0.37502290308475494,0.48529433608055117,0.3857895012944937 +fresh-zh-2,18,19,Q,weak_half,200,0.04343586973845959,0.0534112099558115,0.06460162252187727,0.054462301265448335,200,0.12069911807775498,0.14972881227731705,0.18620306849479676,0.15217649560421706 +fresh-zh-2,18,19,Q,strong_half,200,0.10141547843813896,0.13778479397296906,0.18353935778141023,0.14059118572622537,200,0.27551740407943726,0.3472747951745987,0.43447560667991636,0.35322576962411406 +fresh-zh-2,18,19,Q,random_half,200,0.0674538180232048,0.10073267295956612,0.14462182223796843,0.10473861441016197,200,0.1838500663638115,0.2529953122138977,0.33715625703334806,0.2566286628693342 +fresh-zh-2,18,19,Q,all_heads,200,0.11009455248713493,0.14756211638450623,0.19009753763675688,0.14948608476668596,200,0.40503120720386504,0.5028677880764008,0.6125150680541992,0.5054022626578808 +fresh-zh-2,18,19,K,weak_half,200,0.02139507606625557,0.029337091371417046,0.040903106331825256,0.03025105468928814,200,0.05759431794285774,0.0798911303281784,0.10303315445780754,0.07986554000526666 +fresh-zh-2,18,19,K,strong_half,200,0.06630893573164941,0.09404531121253967,0.12597691863775254,0.09548422431573272,200,0.1729764074087143,0.23908135294914246,0.31214395463466643,0.23949552342295646 +fresh-zh-2,18,19,K,random_half,200,0.03826248943805695,0.06620984524488449,0.1043273754417896,0.06821277300827205,200,0.09528730288147927,0.15070322901010513,0.2283831000328064,0.15793457847088577 +fresh-zh-2,18,19,K,all_heads,200,0.07030560746788979,0.09702940285205841,0.1309701606631279,0.1001494774222374,200,0.24221688657999038,0.3145208805799484,0.4046083450317382,0.319361065030098 +fresh-zh-2,18,19,V,weak_half,200,0.027164516784250737,0.03675483912229538,0.057930187508463835,0.04075504462234676,200,0.0675201341509819,0.08945804834365845,0.1443285077810287,0.10256066888570786 +fresh-zh-2,18,19,V,strong_half,200,0.07111474126577377,0.09883755445480347,0.14024595916271204,0.10836533576250076,200,0.14929614067077637,0.19957126677036285,0.2871866971254349,0.22022919230163096 +fresh-zh-2,18,19,V,random_half,200,0.040711766481399535,0.07158558443188667,0.12034551650285719,0.0768967984430492,200,0.09848215728998184,0.14898236095905304,0.23275760114192962,0.16332753844559192 +fresh-zh-2,18,19,V,all_heads,200,0.084201879799366,0.12034821510314941,0.17330206036567686,0.12868890210986136,200,0.21946438252925873,0.28039059042930603,0.41771781444549555,0.3227898621559143 +fresh-zh-2,18,19,QKV,weak_half,200,0.03474302478134632,0.042306067422032356,0.05446412712335586,0.04368477962911129,200,0.09379126504063606,0.11436434090137482,0.14681384712457657,0.11792435608804226 +fresh-zh-2,18,19,QKV,strong_half,200,0.09066921696066857,0.1150595024228096,0.14625575244426728,0.11780251979827881,200,0.22723627537488938,0.28123360872268677,0.3523650288581848,0.2900273895263672 +fresh-zh-2,18,19,QKV,random_half,200,0.0559904433786869,0.08273791894316673,0.12146416977047919,0.08630330294370651,200,0.14345042854547502,0.19500748068094254,0.2735179543495178,0.20557381115853787 +fresh-zh-2,18,19,QKV,all_heads,200,0.09869400337338448,0.12590667605400085,0.16087260544300078,0.12922039084136486,200,0.3294968128204346,0.3965512663125992,0.4856201529502869,0.407951745390892 +fresh-zh-2,18,19,JS,weak_half,200,0.00024250521382782614,0.0005920730181969702,0.0012649440206587313,0.0006980208386266895,200,0.000649251148570329,0.0026364059885963798,0.006882245838642119,0.0032873435968940614 +fresh-zh-2,18,19,JS,strong_half,200,0.0022280937992036343,0.005862757097929716,0.011078207287937403,0.006152294221392367,200,0.002102449256926775,0.004929718328639865,0.00996520360931754,0.005532952744688373 +fresh-zh-2,18,19,JS,random_half,200,0.001011188398115337,0.0033537927083671093,0.007716056425124407,0.004045719321584329,200,0.0012704378343187273,0.003600375377573073,0.00897225495427847,0.004471481308573857 +fresh-zh-2,18,19,JS,all_heads,200,0.0023725082632154227,0.005990954115986824,0.012146112415939568,0.006737736810464412,200,0.0032219047425314784,0.008727352600544691,0.014426814951002597,0.008820296396734192 +fresh-zh-2,18,19,sqrtJS,weak_half,200,0.0155707354657352,0.024332521483302116,0.03556554354727268,0.025163842444308103,200,0.05689027942717075,0.10115215554833412,0.1405474841594696,0.09954202460125089 +fresh-zh-2,18,19,sqrtJS,strong_half,200,0.04720268733799458,0.07656863704323769,0.10525297746062279,0.0751774178724736,200,0.11308238953351975,0.1763361170887947,0.2307112067937851,0.17409001372754573 +fresh-zh-2,18,19,sqrtJS,random_half,200,0.03179886974394321,0.05791192874312401,0.08784108683466911,0.059726942144334313,200,0.08483682870864868,0.13281464576721191,0.19287664592266082,0.138040353897959 +fresh-zh-2,18,19,sqrtJS,all_heads,200,0.048708079382777214,0.07740125432610512,0.11020908877253532,0.07880721747875213,200,0.18019915819168092,0.2776152193546295,0.35546948909759524,0.27363203816115855 +fresh-zh-2,19,20,Q,weak_half,400,0.029132733307778838,0.06918829306960106,0.13049198389053346,0.07559249960817396,400,0.053592193871736526,0.13652169704437256,0.23861647099256517,0.14287836025469006 +fresh-zh-2,19,20,Q,strong_half,400,0.11178709119558335,0.22989173978567123,0.3729822188615799,0.23967631095089018,400,0.15831009596586226,0.361398383975029,0.5898820757865908,0.36812510961666706 +fresh-zh-2,19,20,Q,random_half,400,0.06500959470868112,0.14637108892202377,0.29187245666980743,0.16533657990396022,400,0.09886630773544312,0.23669524490833282,0.45918326377868657,0.2623799960222095 +fresh-zh-2,19,20,Q,all_heads,400,0.1326583057641983,0.26024550199508667,0.4433286219835282,0.2732466580625623,400,0.2150837391614914,0.5137318670749664,0.8095863044261933,0.5110034695640207 +fresh-zh-2,19,20,K,weak_half,400,0.0408476360142231,0.07328912243247032,0.1210591122508049,0.07997345893178136,400,0.08353856950998306,0.14471954852342606,0.2295814394950867,0.1507434962783009 +fresh-zh-2,19,20,K,strong_half,400,0.18673255443572997,0.2709898054599762,0.43245043158531193,0.2963900580257177,400,0.2485720306634903,0.3681151121854782,0.5766039788722992,0.3947235548496246 +fresh-zh-2,19,20,K,random_half,400,0.09899016097187996,0.16962932795286179,0.30315494835376744,0.19066455096006393,400,0.15449991375207903,0.26099538803100586,0.40835605263710023,0.27551263391971587 +fresh-zh-2,19,20,K,all_heads,400,0.21346824318170549,0.31590621173381805,0.494912925362587,0.3407771019637585,400,0.3427822917699814,0.5048337876796722,0.7881359398365021,0.5454670513421297 +fresh-zh-2,19,20,V,weak_half,400,0.037958654388785366,0.06792227178812027,0.10931142419576646,0.07179429713636637,400,0.07563537508249282,0.13962407410144806,0.2184183105826378,0.144738550754264 +fresh-zh-2,19,20,V,strong_half,400,0.16598552316427231,0.23928199708461761,0.33850589692592625,0.24984082316979767,400,0.2469910204410553,0.3433382362127304,0.4789356678724289,0.3548479728586972 +fresh-zh-2,19,20,V,random_half,400,0.09464022442698479,0.16405072808265686,0.23412676155567172,0.1656580741982907,400,0.14847421944141387,0.24582026898860931,0.36122040152549745,0.25249335415661334 +fresh-zh-2,19,20,V,all_heads,400,0.1804520830512047,0.2717948406934738,0.39794742465019234,0.2848770249821246,400,0.32972692251205443,0.4844033122062683,0.6789735853672028,0.49958652328699826 +fresh-zh-2,19,20,QKV,weak_half,400,0.04618144631385803,0.0746610201895237,0.11451776549220086,0.07939961397089064,400,0.09418824911117554,0.14693641662597656,0.22121053487062456,0.15290793740190567 +fresh-zh-2,19,20,QKV,strong_half,400,0.1971924602985382,0.2603868544101715,0.36527378261089327,0.27256594467908146,400,0.2800373196601868,0.3833204209804535,0.5236369490623474,0.3922309302911162 +fresh-zh-2,19,20,QKV,random_half,400,0.1098603643476963,0.17117168754339218,0.27860574424266815,0.18220270670950411,400,0.16591839194297792,0.26518554985523224,0.4035797506570816,0.2769140244647861 +fresh-zh-2,19,20,QKV,all_heads,400,0.22002118974924087,0.29310372471809387,0.42014084458351136,0.31062665704637765,400,0.38525274097919465,0.5362839102745056,0.7381349325180054,0.5451388680562377 +fresh-zh-2,20,21,Q,weak_half,400,0.009668495412915945,0.02057089749723673,0.04933476373553276,0.02505562474601902,400,0.02570768613368273,0.055161723867058754,0.1283269375562668,0.06622660318389535 +fresh-zh-2,20,21,Q,strong_half,400,0.03379235304892063,0.056567639112472534,0.15362230390310289,0.07609079562593252,400,0.07468534409999848,0.1295669749379158,0.37744880020618454,0.1743573955539614 +fresh-zh-2,20,21,Q,random_half,400,0.02053077630698681,0.03940158151090145,0.10495732799172403,0.05105617424938828,400,0.04919772818684578,0.09013020992279053,0.2428750962018967,0.11856725201942027 +fresh-zh-2,20,21,Q,all_heads,400,0.03569026812911034,0.05932460352778435,0.15747668743133547,0.07973034649621695,400,0.10304542332887649,0.18540286272764206,0.5109435737133026,0.24058399939909578 +fresh-zh-2,20,21,K,weak_half,400,0.019436262920498847,0.03360874205827713,0.052350670471787465,0.03491121207596734,400,0.05709293149411678,0.09076329320669174,0.13480032235383993,0.09397784498520195 +fresh-zh-2,20,21,K,strong_half,400,0.06183897182345391,0.10398082435131073,0.15442621260881426,0.10941727365367114,400,0.14731434881687164,0.24462030082941055,0.3468039721250535,0.2531783849932253 +fresh-zh-2,20,21,K,random_half,400,0.04018903858959675,0.0703829750418663,0.11401844546198846,0.07607082056347281,400,0.09496193826198578,0.16491154581308365,0.2469186082482338,0.17188335737213492 +fresh-zh-2,20,21,K,all_heads,400,0.06598936170339584,0.11039680615067482,0.159928759932518,0.11542578745633364,400,0.21153693348169328,0.3370554894208908,0.47224041521549226,0.34715622941032054 +fresh-zh-2,20,21,V,weak_half,400,0.02335045747458935,0.03405425697565079,0.05479963310062885,0.03664177108788863,400,0.06184705495834351,0.0871032141149044,0.13995280265808105,0.09523092238232493 +fresh-zh-2,20,21,V,strong_half,400,0.06501398757100106,0.10607532411813736,0.1782878130674362,0.1137263381946832,400,0.18295613676309586,0.25268012285232544,0.3471831977367401,0.2625129249691963 +fresh-zh-2,20,21,V,random_half,400,0.040927334874868396,0.06918914243578911,0.12362730428576471,0.07644475983455777,400,0.11473528444766999,0.16728563606739044,0.26212739944458013,0.17899662056937815 +fresh-zh-2,20,21,V,all_heads,400,0.06994557976722718,0.11144682765007019,0.18708560466766358,0.12013503015972674,400,0.25448001027107237,0.3373476713895798,0.4810174673795701,0.35774384658783676 +fresh-zh-2,20,21,QKV,weak_half,400,0.022813315875828265,0.03220172971487045,0.047823506593704226,0.033984517608769235,400,0.06008492782711983,0.08833202719688416,0.12252889797091485,0.09026636964641511 +fresh-zh-2,20,21,QKV,strong_half,400,0.07372181937098503,0.10217367857694626,0.13863164335489273,0.10640455747023225,400,0.1860516905784607,0.23195207864046097,0.33120594918727875,0.24746867146342993 +fresh-zh-2,20,21,QKV,random_half,400,0.04499485492706299,0.06841224804520607,0.10116147100925446,0.07272061202675104,400,0.11059885397553444,0.15775678306818008,0.22803703546524054,0.1675259772129357 +fresh-zh-2,20,21,QKV,all_heads,400,0.0800431177020073,0.10726960748434067,0.1462291657924652,0.112245813915506,400,0.24738598614931107,0.32032592594623566,0.4463034331798553,0.3377350413799286 +fresh-zh-2,21,22,Q,weak_half,400,0.010180623829364776,0.02361456211656332,0.03672200217843056,0.023857224310049788,400,0.026514287292957305,0.06644105911254883,0.10242440253496171,0.06698753490578384 +fresh-zh-2,21,22,Q,strong_half,400,0.017551238276064397,0.043949637562036514,0.10316911712288859,0.05734742910601199,400,0.042677752673625946,0.11785091087222099,0.2405280351638794,0.1327750119380653 +fresh-zh-2,21,22,Q,random_half,400,0.013245817366987468,0.032493073493242264,0.06269722729921341,0.04000680840923451,400,0.036257249489426616,0.0924634039402008,0.1554543524980545,0.09987098366953433 +fresh-zh-2,21,22,Q,all_heads,400,0.021399239264428615,0.054019488394260406,0.1109355114400387,0.06665257172659039,400,0.06882400140166282,0.18573115766048431,0.3417895436286926,0.19976254670880736 +fresh-zh-2,21,22,K,weak_half,400,0.015017369203269484,0.02954161260277033,0.05137990228831769,0.03186574855586514,400,0.042855098843574524,0.08116276562213898,0.1303072676062584,0.0837585540721193 +fresh-zh-2,21,22,K,strong_half,400,0.02680951040238142,0.055364418774843216,0.08767663016915322,0.056696253777481616,400,0.07750426232814789,0.1491631120443344,0.23081339597702027,0.15277843412943184 +fresh-zh-2,21,22,K,random_half,400,0.022574311308562755,0.045013850554823875,0.07654797881841663,0.047533776904456315,400,0.05646455883979797,0.11455470323562622,0.18064719587564468,0.11789412103127689 +fresh-zh-2,21,22,K,all_heads,400,0.031951555609703065,0.06385228782892227,0.0982666037976742,0.06517283228226006,400,0.12104344516992568,0.23283731192350388,0.35780125260353096,0.23653698806650936 +fresh-zh-2,21,22,V,weak_half,400,0.017767232656478883,0.02797630988061428,0.045421451330184937,0.029956487389281392,400,0.05159397646784782,0.07418710738420486,0.11415400579571724,0.07959780611097812 +fresh-zh-2,21,22,V,strong_half,400,0.037131933495402336,0.057205693796277046,0.08601602911949158,0.05960198762826621,400,0.0980858750641346,0.14578591287136078,0.21819515526294708,0.15301066068932415 +fresh-zh-2,21,22,V,random_half,400,0.026875473186373712,0.04335791431367397,0.07130331322550774,0.046637262939475475,400,0.07250906080007553,0.10856769606471062,0.17079934179782869,0.11577062497846782 +fresh-zh-2,21,22,V,all_heads,400,0.04362951517105103,0.0674946941435337,0.09832716584205628,0.06930900503415614,400,0.15004875659942626,0.22157417237758636,0.3262884736061097,0.23260846806690097 +fresh-zh-2,21,22,QKV,weak_half,400,0.019261209480464458,0.02859506942331791,0.04173603765666485,0.029872331181541086,400,0.0558157954365015,0.07750311121344566,0.10956613570451737,0.08048618629574776 +fresh-zh-2,21,22,QKV,strong_half,400,0.038578963652253155,0.05508006736636162,0.08402577638626099,0.061800146354362366,400,0.10234399884939194,0.1462731584906578,0.20761048942804347,0.15610017945989965 +fresh-zh-2,21,22,QKV,random_half,400,0.02802417017519474,0.04343114793300629,0.06491640284657478,0.047637422611005606,400,0.07781179249286652,0.11295495182275772,0.15947679430246353,0.11806187251582742 +fresh-zh-2,21,22,QKV,all_heads,400,0.04610181860625744,0.06492195278406143,0.09259439557790762,0.07107665246352553,400,0.15914543122053146,0.2251080498099327,0.3163329422473908,0.2365863665379584 +fresh-zh-2,22,23,Q,weak_half,200,0.03337487690150738,0.04328298941254616,0.053113381192088124,0.04332227567210793,200,0.08538906127214432,0.11311951279640198,0.14269760847091675,0.11423501297831536 +fresh-zh-2,22,23,Q,strong_half,200,0.11341480389237404,0.1499331295490265,0.2048303782939911,0.15525514360517265,200,0.2666120707988739,0.33727671205997467,0.4539600223302841,0.3498956037312746 +fresh-zh-2,22,23,Q,random_half,200,0.06507262662053108,0.10269999131560326,0.14682760685682297,0.10607084294781088,200,0.1623355984687805,0.2225237935781479,0.3249312490224838,0.2331556247547269 +fresh-zh-2,22,23,Q,all_heads,200,0.12089833170175553,0.15739864110946655,0.21130090355873107,0.16323967907577752,200,0.36630937457084656,0.4459937661886215,0.5909654498100281,0.46413061559200286 +fresh-zh-2,22,23,K,weak_half,200,0.026410561427474024,0.03551407530903816,0.049590005725622166,0.035703546069562435,200,0.06431902199983597,0.0851285420358181,0.11872883290052413,0.08853252543136478 +fresh-zh-2,22,23,K,strong_half,200,0.09916138574481011,0.15099727362394333,0.24074922204017635,0.16160346873104572,200,0.2680500030517578,0.3733121156692505,0.5108372092247009,0.3749830161035061 +fresh-zh-2,22,23,K,random_half,200,0.05179771184921265,0.09568056464195251,0.200287763774395,0.11124949980527163,200,0.12257678210735322,0.2196837067604065,0.3533358633518219,0.23148766733705997 +fresh-zh-2,22,23,K,all_heads,200,0.1047767773270607,0.15549564361572266,0.2397261828184127,0.16475559026002884,200,0.34670293331146246,0.45955072343349457,0.6408070564270019,0.46351554334163664 +fresh-zh-2,22,23,V,weak_half,200,0.015303421765565872,0.018753248266875744,0.0236631091684103,0.01940230854321271,200,0.038052551448345184,0.049823157489299774,0.05879689678549766,0.04995307855308056 +fresh-zh-2,22,23,V,strong_half,200,0.061385627835989,0.10212643072009087,0.242867225408554,0.12938223291188478,200,0.1875476583838463,0.24866582453250885,0.44159367680549616,0.2874128333479166 +fresh-zh-2,22,23,V,random_half,200,0.03207519166171551,0.06526782736182213,0.19970931261777877,0.08799737889319659,200,0.08021941930055619,0.14230583608150482,0.32452421486377714,0.16868179470300673 +fresh-zh-2,22,23,V,all_heads,200,0.06421406045556069,0.10421813279390335,0.2227419123053549,0.12853015922009944,200,0.23345310240983966,0.29939574003219604,0.4970016241073608,0.3373659113049507 +fresh-zh-2,22,23,QKV,weak_half,200,0.02718850299715996,0.03372202813625336,0.04219317361712455,0.034473660262301566,200,0.06837436631321907,0.08775146678090096,0.10935811921954156,0.08936635391786694 +fresh-zh-2,22,23,QKV,strong_half,200,0.10299415811896324,0.14419133961200714,0.2105528876185417,0.1535345297679305,200,0.2643497049808502,0.3496076762676239,0.4501960456371307,0.35508953779935837 +fresh-zh-2,22,23,QKV,random_half,200,0.053894609212875366,0.09474949911236763,0.18336668461561204,0.10536875719204546,200,0.13231368213891984,0.2064884975552559,0.34456894993782045,0.2224882211908698 +fresh-zh-2,22,23,QKV,all_heads,200,0.11056995540857316,0.15081188082695007,0.2133706957101822,0.1569971214979887,200,0.3383635878562927,0.43763378262519836,0.5444308936595916,0.44445588991045953 +fresh-zh-2,22,23,JS,weak_half,200,1.4770010056963657e-05,0.0002922067360486835,0.0007405725715216249,0.0003350366416108841,200,1.9988986787211614e-05,0.0003214960888726637,0.0015551407239399843,0.0006810629858841821 +fresh-zh-2,22,23,JS,strong_half,200,0.0004930023133056238,0.006734809605404735,0.01643774602562189,0.008211456818289662,200,0.0006905986985657366,0.006920999381691217,0.01484781568869948,0.00754679398254666 +fresh-zh-2,22,23,JS,random_half,200,0.0003158875450026244,0.005039972020313144,0.019370814226567744,0.007948227855813457,200,0.000284543022280559,0.0031657430808991194,0.00942688062787056,0.004173842780714949 +fresh-zh-2,22,23,JS,all_heads,200,0.0005239922378677876,0.006753074238076806,0.01669903229922056,0.008064929709125862,200,0.0007367950398474933,0.007722364738583565,0.016095035895705222,0.008227856991215959 +fresh-zh-2,22,23,sqrtJS,weak_half,200,0.0038431539898738268,0.01709404680877924,0.027213462442159653,0.01625037259305827,200,0.010478159505873918,0.044305017217993736,0.07230504527688027,0.04362094839802012 +fresh-zh-2,22,23,sqrtJS,strong_half,200,0.02220356371253729,0.08206578344106674,0.12820975333452225,0.08034256451996043,200,0.05698655545711518,0.18434789776802063,0.2532596707344055,0.17324928887188434 +fresh-zh-2,22,23,sqrtJS,random_half,200,0.01777321510016918,0.07098878175020218,0.13917901366949081,0.0767205052031204,200,0.033352289348840714,0.11237890273332596,0.18021265268325803,0.11028609457425773 +fresh-zh-2,22,23,sqrtJS,all_heads,200,0.022889291681349278,0.08217687904834747,0.12922449558973312,0.08049048298504204,200,0.07260303720831872,0.2369401752948761,0.30805366337299345,0.2168702360801399 +fresh-en-1,0,1,Q,weak_half,400,0.1897168591618538,0.29871125519275665,0.4515512138605118,0.3121999656222761,400,0.4314333647489548,0.6547353863716125,0.9448965013027193,0.6772705401480198 +fresh-en-1,0,1,Q,strong_half,400,0.7236248672008514,1.0577986240386963,1.440848469734192,1.075602947473526,400,1.700400745868683,2.809340715408325,3.8353605747222903,2.7923788911104204 +fresh-en-1,0,1,Q,random_half,400,0.5268789172172547,1.0194953680038452,1.3797268509864808,0.9937241809815168,400,0.8819768071174622,1.6079294085502625,2.653616809844971,1.6960216903686522 +fresh-en-1,0,1,Q,all_heads,400,0.9413664519786835,1.2939398288726807,1.5878896594047547,1.2734367954730987,400,2.234930944442749,3.531088352203369,4.652729415893555,3.469649436175823 +fresh-en-1,0,1,K,weak_half,400,0.2248666375875473,0.33381591737270355,0.5401574969291687,0.36333866853266955,400,0.5345231413841247,0.7339546382427216,1.0744285225868226,0.7708222552388907 +fresh-en-1,0,1,K,strong_half,400,0.7613664686679841,1.0323171615600586,1.3784624457359316,1.050206831842661,400,2.0293044328689573,2.6711169481277466,3.6607686281204224,2.7541363286972045 +fresh-en-1,0,1,K,random_half,400,0.5932540357112884,0.9620604515075684,1.3080284714698793,0.9651395785063506,400,1.0974225521087646,1.6849610209465027,2.4564747810363774,1.7348233909904958 +fresh-en-1,0,1,K,all_heads,400,1.0180368185043336,1.2842377424240112,1.5322563171386718,1.2731784419715404,400,2.6672797441482543,3.425931215286255,4.516887521743775,3.5249585929512977 +fresh-en-1,0,1,V,weak_half,400,0.1083014577627182,0.24108338356018066,0.511081874370575,0.2802542892936617,400,0.3092775344848633,0.5678510069847107,0.9418403446674348,0.5982575616240502 +fresh-en-1,0,1,V,strong_half,400,0.4971258133649826,0.8670008480548859,1.3737720489501954,0.9332739974185824,400,1.2920219540596007,2.143291711807251,3.7844115018844624,2.3718513211607934 +fresh-en-1,0,1,V,random_half,400,0.3280008554458618,0.7320268154144287,1.6326180696487436,0.900690052807331,400,0.7277421176433563,1.2474647164344788,2.342020750045777,1.4403877830505372 +fresh-en-1,0,1,V,all_heads,400,0.6487612903118134,1.0565130114555359,1.6007157564163208,1.1013702577352524,400,1.6181118845939637,2.6892964839935303,4.556001949310304,2.970108874589205 +fresh-en-1,0,1,QKV,weak_half,400,0.22440937012434006,0.30879026651382446,0.4747848123311997,0.3339493476971984,400,0.5418076276779176,0.6916033029556274,0.9337150394916535,0.7180722783505916 +fresh-en-1,0,1,QKV,strong_half,400,0.7986591935157776,1.0134631991386414,1.303740990161896,1.04798963829875,400,2.0846874713897705,2.6961590051651,3.5494216203689577,2.8023727306723596 +fresh-en-1,0,1,QKV,random_half,400,0.6446394562721253,0.9517265558242798,1.3522567749023438,0.997944149300456,400,1.1141008615493775,1.601073980331421,2.478505063056946,1.7217424581944942 +fresh-en-1,0,1,QKV,all_heads,400,1.0098045229911805,1.2264060974121094,1.4670547842979436,1.2373458036780358,400,2.7346471309661866,3.379374384880066,4.430821275711059,3.5204450070858 +fresh-en-1,1,2,Q,weak_half,400,0.0393075667321682,0.08068036660552025,0.14861145913600923,0.08981951339635998,400,0.08939508497714997,0.19545432925224304,0.3704800844192505,0.21698016277514398 +fresh-en-1,1,2,Q,strong_half,400,0.17585757225751877,0.3716534823179245,0.6688020348548889,0.4078899082168937,400,0.3763920396566391,0.8159739375114441,1.3025718212127688,0.8490002179518342 +fresh-en-1,1,2,Q,random_half,400,0.10905859544873238,0.22016847878694534,0.46401039361953755,0.26705049963667987,400,0.2287642389535904,0.4824535846710205,0.8908980906009675,0.5376374381966889 +fresh-en-1,1,2,Q,all_heads,400,0.20248614102602006,0.4223291575908661,0.7281758785247803,0.44372582765296104,400,0.4932699054479599,1.0213602185249329,1.6281457662582397,1.065980382859707 +fresh-en-1,1,2,K,weak_half,400,0.05401750169694424,0.10523257777094841,0.20942885428667068,0.1206287502963096,400,0.12815450578927995,0.24141622334718704,0.4005704402923585,0.25789196491241456 +fresh-en-1,1,2,K,strong_half,400,0.25792816281318665,0.40744054317474365,0.680982780456543,0.4379813690856099,400,0.6190192461013794,0.8974402844905853,1.4151566267013553,0.9629573537409306 +fresh-en-1,1,2,K,random_half,400,0.1521135225892067,0.27063708007335663,0.4935710638761522,0.30330596562474965,400,0.35227090716362,0.5516796708106995,0.928322505950928,0.6118973575159907 +fresh-en-1,1,2,K,all_heads,400,0.28382281959056854,0.4604008346796036,0.7198263108730316,0.48403409563004973,400,0.7984650254249573,1.1629895567893982,1.7354826569557191,1.2208493204414845 +fresh-en-1,1,2,V,weak_half,400,0.03510716371238232,0.09583758935332298,0.2510539084672928,0.12077475939644501,400,0.07688825279474258,0.1936376467347145,0.4436574608087541,0.2356573272496462 +fresh-en-1,1,2,V,strong_half,400,0.186335888504982,0.37076884508132935,0.7481388628482819,0.435244386298582,400,0.40786643922328947,0.8623624742031097,1.627833330631256,0.9503131012246012 +fresh-en-1,1,2,V,random_half,400,0.10513567999005317,0.24889563769102097,0.5485041081905367,0.2977012393344194,400,0.2475702330470085,0.5165702998638153,1.041429615020752,0.5940332181751729 +fresh-en-1,1,2,V,all_heads,400,0.20049792379140854,0.40690886974334717,0.8376210212707521,0.48317090155556797,400,0.5248587846755982,1.081919252872467,1.977030086517334,1.1859704278409482 +fresh-en-1,1,2,QKV,weak_half,400,0.05690940283238888,0.10242142900824547,0.20097360014915472,0.11696762687526643,400,0.12432339414954187,0.23649076372385025,0.4004278481006624,0.2525361436791718 +fresh-en-1,1,2,QKV,strong_half,400,0.26677471697330474,0.4161740690469742,0.6378061890602112,0.44623036615550515,400,0.6289057254791259,0.9122443795204163,1.4276185512542725,0.9877189197391272 +fresh-en-1,1,2,QKV,random_half,400,0.15969948023557662,0.2721346318721771,0.4990270286798477,0.30513102140277626,400,0.36363035440444946,0.5619656145572662,0.9524280369281769,0.6231165655329823 +fresh-en-1,1,2,QKV,all_heads,400,0.2925855845212937,0.4659601151943207,0.7179368138313293,0.4915473873540759,400,0.799589091539383,1.1612361669540405,1.770927929878235,1.2402550715208054 +fresh-en-1,2,3,Q,weak_half,200,0.05517833791673184,0.09756537154316902,0.16837549805641175,0.10516038900241256,200,0.12082152962684632,0.22928452491760254,0.42202447950839994,0.24884872209280728 +fresh-en-1,2,3,Q,strong_half,200,0.42676768600940707,0.5721633434295654,0.7386204719543457,0.576854162812233,200,0.7865302443504334,1.0911297798156738,1.3966598629951477,1.088696400821209 +fresh-en-1,2,3,Q,random_half,200,0.16998187601566314,0.3429967910051346,0.5257752418518066,0.3472297453135252,200,0.3579265624284744,0.6499396562576294,0.9944325387477875,0.6618276499211788 +fresh-en-1,2,3,Q,all_heads,200,0.4558986246585846,0.6176938116550446,0.7579807460308075,0.6193193340301514,200,0.948799604177475,1.314595878124237,1.7647330045700074,1.337545118033886 +fresh-en-1,2,3,K,weak_half,200,0.06204530447721482,0.09623423963785172,0.15658432692289354,0.10269248824566603,200,0.1254363223910332,0.2145834043622017,0.3811819553375244,0.24036242917180062 +fresh-en-1,2,3,K,strong_half,200,0.3842017978429794,0.48823413252830505,0.5778945088386536,0.4865820972621441,200,0.8544061601161957,1.0253400206565857,1.3228937864303585,1.067280252277851 +fresh-en-1,2,3,K,random_half,200,0.19634587168693543,0.31035327911376953,0.4448583573102951,0.3169101969152689,200,0.4199701279401779,0.6321394145488739,0.8644249558448791,0.6436873100697994 +fresh-en-1,2,3,K,all_heads,200,0.4236966043710709,0.5291590690612793,0.5960596382617951,0.5202633786201477,200,1.0109024047851562,1.2307273149490356,1.6094459414482114,1.3076426923274993 +fresh-en-1,2,3,V,weak_half,200,0.08598067164421082,0.12154579907655716,0.19041833430528635,0.13170458815991878,200,0.1776464819908142,0.28206655383110046,0.46513289213180514,0.3088893750309944 +fresh-en-1,2,3,V,strong_half,200,0.40890307128429415,0.5036363005638123,0.7611819684505462,0.544224812835455,200,0.9397101879119873,1.0860215425491333,1.6904223680496215,1.2290874686837197 +fresh-en-1,2,3,V,random_half,200,0.21016969233751298,0.3482092618942261,0.5576824307441711,0.37171134367585185,200,0.4757844179868698,0.7053122520446777,1.1137233972549438,0.7624429002404213 +fresh-en-1,2,3,V,all_heads,200,0.4528144598007202,0.54301318526268,0.808084452152252,0.5948349463939667,200,1.1749778985977173,1.4640721678733826,2.2168956995010376,1.5379768490791321 +fresh-en-1,2,3,QKV,weak_half,200,0.07116407379508019,0.11017919704318047,0.17107924669981,0.11480151131749153,200,0.14969091564416886,0.24277716875076294,0.43449330925941465,0.2711820526048541 +fresh-en-1,2,3,QKV,strong_half,200,0.42104344069957733,0.5248804688453674,0.6927355647087097,0.5407563364505767,200,0.9188218653202057,1.092450737953186,1.4578927040100098,1.1460741448402405 +fresh-en-1,2,3,QKV,random_half,200,0.20177502632141114,0.3453436493873596,0.5051529288291932,0.3490396623685956,200,0.4372090846300125,0.672469824552536,1.014020538330078,0.7007229068875312 +fresh-en-1,2,3,QKV,all_heads,200,0.4583275020122528,0.5678347051143646,0.7255826950073242,0.5831889589130879,200,1.1010741233825683,1.3543728590011597,1.8310943961143493,1.4172562089562417 +fresh-en-1,2,3,JS,weak_half,200,0.001327572239097208,0.0034615908516570926,0.008200257178395986,0.004260162497375859,200,0.0018504797015339138,0.00457689119502902,0.010083642601966857,0.005431857280782424 +fresh-en-1,2,3,JS,strong_half,200,0.033670518547296524,0.06746451929211617,0.12440710365772246,0.07815434897784143,200,0.03213831000030041,0.05883792042732239,0.1186465762555599,0.07047672783955931 +fresh-en-1,2,3,JS,random_half,200,0.02028951179236174,0.044922368600964546,0.08441445678472519,0.05095427403226495,200,0.006119565432891249,0.0234531806781888,0.08358347564935684,0.0362449056917103 +fresh-en-1,2,3,JS,all_heads,200,0.03989506028592587,0.08069420605897903,0.14209602773189545,0.08931239541154355,200,0.036579983308911324,0.0637592300772667,0.13071855455636977,0.07590858483687044 +fresh-en-1,2,3,sqrtJS,weak_half,200,0.03643577992916107,0.05883447267115116,0.09055527225136757,0.061199579280801115,200,0.11236283034086228,0.17698857188224792,0.25103408098220825,0.1758392832428217 +fresh-en-1,2,3,sqrtJS,strong_half,200,0.18349353522062303,0.25973929464817047,0.35271372497081754,0.2698957652598619,200,0.4115959137678146,0.5413856208324432,0.7561820924282073,0.562640193104744 +fresh-en-1,2,3,sqrtJS,random_half,200,0.14244107604026796,0.2119487226009369,0.29054144620895384,0.21661524347960948,200,0.19256483018398285,0.3562793880701065,0.5318841934204102,0.36070496883243325 +fresh-en-1,2,3,sqrtJS,all_heads,200,0.1997360661625862,0.2840672582387924,0.37695626318454745,0.28998637303709984,200,0.5252659022808075,0.7200451493263245,0.9775034844875335,0.7384794771671295 +fresh-en-1,3,4,Q,weak_half,400,0.10325723364949227,0.19835466146469116,0.3620797008275986,0.22094974648207427,400,0.21281687617301942,0.40390531718730927,0.6646402657032014,0.4238525830581784 +fresh-en-1,3,4,Q,strong_half,400,0.14759760349988937,0.28940635919570923,0.6664236783981328,0.3583635596372187,400,0.3966772824525833,0.7561030685901642,2.238220477104187,0.9996034225076437 +fresh-en-1,3,4,Q,random_half,400,0.13277187198400497,0.2510734796524048,0.5904966294765475,0.3177546208538115,400,0.29222558438777924,0.5782100856304169,1.2599046587944032,0.7103668061643839 +fresh-en-1,3,4,Q,all_heads,400,0.15904244482517244,0.31065821647644043,0.7169106781482698,0.3799043721705675,400,0.6302353322505951,1.1633158922195435,2.6859280109405517,1.4234560080617666 +fresh-en-1,3,4,K,weak_half,400,0.1356964498758316,0.2044411599636078,0.32888354361057287,0.2245290471613407,400,0.26541247963905334,0.38793426752090454,0.5973719298839569,0.41127674613147974 +fresh-en-1,3,4,K,strong_half,400,0.1809505119919777,0.2658175826072693,0.45535756051540377,0.2938645518012345,400,0.48423089981079104,0.7314157485961914,1.2559667468070987,0.8030020435154438 +fresh-en-1,3,4,K,random_half,400,0.1672549918293953,0.26152630150318146,0.4776602029800416,0.2904001390747726,400,0.3691681802272797,0.5516111850738525,0.9326141476631171,0.6072892582416535 +fresh-en-1,3,4,K,all_heads,400,0.2046289399266243,0.29176121950149536,0.4951414704322817,0.3306192489154637,400,0.7813272476196289,1.111222743988037,1.8391676306724551,1.214278787970543 +fresh-en-1,3,4,V,weak_half,400,0.08516281768679619,0.1824745312333107,0.3937461376190186,0.2203760804887861,400,0.19219243228435517,0.3361160308122635,0.6609165370464326,0.39243775874376297 +fresh-en-1,3,4,V,strong_half,400,0.11053132191300392,0.24657996743917465,0.4539019525051118,0.273958642296493,400,0.40910349488258363,0.7498108148574829,1.397873389720917,0.835242274031043 +fresh-en-1,3,4,V,random_half,400,0.08943467587232591,0.23394657671451569,0.499888586997986,0.2663247088249773,400,0.2785165995359421,0.5541070103645325,1.0671089172363284,0.6061700121313334 +fresh-en-1,3,4,V,all_heads,400,0.11509588733315468,0.2506822645664215,0.5112366795539857,0.29404366644099356,400,0.6016672372817993,1.0942686796188354,2.0394407033920294,1.2276800353825092 +fresh-en-1,3,4,QKV,weak_half,400,0.1332392528653145,0.203879252076149,0.3710781306028367,0.23082210125401617,400,0.2711304843425751,0.3946233540773392,0.6282978117465973,0.4262784805893898 +fresh-en-1,3,4,QKV,strong_half,400,0.18622651249170302,0.2945380210876465,0.5203353881835937,0.32881732933223246,400,0.5193298816680908,0.8152539432048798,1.561824631690979,0.9395674996823072 +fresh-en-1,3,4,QKV,random_half,400,0.16885371506214142,0.27100829780101776,0.5177827060222626,0.3104778637178242,400,0.38223399519920354,0.6002301275730133,1.0806801080703736,0.6795680347457528 +fresh-en-1,3,4,QKV,all_heads,400,0.20134260952472688,0.3082532286643982,0.568367213010788,0.35443850185722114,400,0.8022432863712311,1.2332290410995483,2.080500626564026,1.3658459788560868 +fresh-en-1,4,5,Q,weak_half,400,0.05627318322658539,0.10497367009520531,0.16213164329528812,0.10903269360307605,400,0.17173204869031908,0.2835026681423187,0.43763933777809144,0.29555343778803944 +fresh-en-1,4,5,Q,strong_half,400,0.13218896090984344,0.2527988404035568,0.4159394323825837,0.26690780533477665,400,0.39622758924961093,0.7376688122749329,1.1061098456382752,0.7531960269063711 +fresh-en-1,4,5,Q,random_half,400,0.09688356146216394,0.181419275701046,0.31556214094161994,0.1970193797443062,400,0.2679353475570679,0.5102981328964233,0.8109845817089081,0.5214784245193005 +fresh-en-1,4,5,Q,all_heads,400,0.14331170469522478,0.2701638489961624,0.44959621429443364,0.2858622779883444,400,0.5661739468574524,1.0335242748260498,1.5487374663352966,1.048749465495348 +fresh-en-1,4,5,K,weak_half,400,0.06250731460750103,0.1091088280081749,0.1686831533908844,0.1128610932501033,400,0.16729704886674882,0.27999818325042725,0.4132414013147354,0.2876661484781653 +fresh-en-1,4,5,K,strong_half,400,0.14067930430173875,0.22785299271345139,0.3421857982873917,0.23624590327497572,400,0.45084729194641116,0.6773179173469543,1.0012911558151247,0.6904219353199005 +fresh-en-1,4,5,K,random_half,400,0.10841051489114764,0.17789264023303986,0.2782657444477082,0.1869765759864822,400,0.2882421970367432,0.47590145468711853,0.7019945204257966,0.4836141585931182 +fresh-en-1,4,5,K,all_heads,400,0.15307794809341432,0.2509654462337494,0.3727332532405854,0.259294420061633,400,0.6336847543716431,0.9515199959278107,1.3863349556922913,0.9780880829319358 +fresh-en-1,4,5,V,weak_half,400,0.02729182466864586,0.07898955792188644,0.1769861474633217,0.09314169076271356,400,0.09041332453489304,0.2022455409169197,0.41107873916625975,0.2273049277625978 +fresh-en-1,4,5,V,strong_half,400,0.07053780630230905,0.17069720476865768,0.42348575592041027,0.21283890633378177,400,0.2322392150759697,0.5078048706054688,1.2024192333221437,0.6100180047750473 +fresh-en-1,4,5,V,random_half,400,0.047004817426204694,0.12662048637866974,0.32420971691608436,0.157892127064988,400,0.14833809733390807,0.34959112107753754,0.8170296967029574,0.41778923202306034 +fresh-en-1,4,5,V,all_heads,400,0.07861930057406426,0.18963070213794708,0.4693473607301715,0.23285733424127103,400,0.3309630811214447,0.7136850357055664,1.6051203846931457,0.8373229334130883 +fresh-en-1,4,5,QKV,weak_half,400,0.06384341791272163,0.10350154712796211,0.16378506422042846,0.11017273931764066,400,0.18309116512537002,0.27350863814353943,0.41861734688282026,0.2874329188093543 +fresh-en-1,4,5,QKV,strong_half,400,0.1531897321343422,0.23054015636444092,0.37335946261882785,0.2513449342735112,400,0.48207282125949863,0.6634368896484375,1.0922566652297974,0.7325273017585278 +fresh-en-1,4,5,QKV,random_half,400,0.1125412940979004,0.17665015906095505,0.29815273582935337,0.19033582836389543,400,0.31815101206302643,0.4792124032974243,0.7507936239242554,0.5065596862882376 +fresh-en-1,4,5,QKV,all_heads,400,0.16421259343624114,0.25263479351997375,0.4111855626106262,0.2724244729802012,400,0.676416563987732,0.9491521120071411,1.4874159455299378,1.019960217922926 +fresh-en-1,5,6,Q,weak_half,400,0.0763455830514431,0.12615040689706802,0.19457937479019166,0.1309581674449146,400,0.2292124107480049,0.3567764014005661,0.5104157209396363,0.36428596595302226 +fresh-en-1,5,6,Q,strong_half,400,0.16009031236171722,0.27196115255355835,0.43384082615375524,0.28680048443377015,400,0.5021184355020524,0.7825570106506348,1.0962231040000918,0.7938538047671319 +fresh-en-1,5,6,Q,random_half,400,0.11733966320753098,0.20945154875516891,0.3297173082828522,0.2223551207780838,400,0.36320409476757054,0.5626322627067566,0.8120993971824646,0.5805081163346767 +fresh-en-1,5,6,Q,all_heads,400,0.17445149272680283,0.30340468883514404,0.4814866155385971,0.3173621469549835,400,0.7682637453079224,1.15475994348526,1.569871735572815,1.1581397669762372 +fresh-en-1,5,6,K,weak_half,400,0.10174274370074272,0.14490167796611786,0.21211867183446884,0.1530913794785738,400,0.27488389015197756,0.3764019310474396,0.515391266345978,0.38831116430461404 +fresh-en-1,5,6,K,strong_half,400,0.19996238052845,0.2699262350797653,0.3779417216777802,0.2812765285745263,400,0.5863300561904907,0.7492568492889404,1.0316694974899292,0.784339925199747 +fresh-en-1,5,6,K,random_half,400,0.15100958049297333,0.22117246687412262,0.3369421660900116,0.23519096991047264,400,0.4060281395912171,0.5674139559268951,0.7863560557365418,0.5834939853847027 +fresh-en-1,5,6,K,all_heads,400,0.22617772966623306,0.3117758482694626,0.43348658084869385,0.32163544051349163,400,0.8824159026145936,1.138687551021576,1.531205701828003,1.1726510955393314 +fresh-en-1,5,6,V,weak_half,400,0.05955852046608925,0.12694881856441498,0.2524970442056656,0.1460492231976241,400,0.16876038610935212,0.3221008628606796,0.4983515650033951,0.32986316926777365 +fresh-en-1,5,6,V,strong_half,400,0.12244098633527756,0.25381462275981903,0.4674642264842988,0.28009930985048415,400,0.4115145742893219,0.7175135314464569,1.144437313079834,0.7588070478662848 +fresh-en-1,5,6,V,random_half,400,0.09878396540880204,0.19105437397956848,0.37898568212985995,0.22021528718061745,400,0.291456937789917,0.49499382078647614,0.8309109807014466,0.5413865588605404 +fresh-en-1,5,6,V,all_heads,400,0.15205028355121614,0.3124743402004242,0.5375273108482362,0.33408756686374547,400,0.600328016281128,1.0609175562858582,1.6198515892028815,1.0886702211946249 +fresh-en-1,5,6,QKV,weak_half,400,0.10360311046242715,0.14396127313375473,0.20555928647518162,0.149974729642272,400,0.2784867167472839,0.3786667287349701,0.4787066459655762,0.379557382427156 +fresh-en-1,5,6,QKV,strong_half,400,0.20638755559921265,0.287594199180603,0.38465026915073397,0.2935511522740126,400,0.630316573381424,0.814426064491272,1.029130458831787,0.8275726975500584 +fresh-en-1,5,6,QKV,random_half,400,0.15547575503587724,0.2274092510342598,0.32100732326507575,0.23590999640524388,400,0.4354691505432129,0.5856751799583435,0.7992608428001404,0.6020987848937511 +fresh-en-1,5,6,QKV,all_heads,400,0.24432264566421508,0.3243936747312546,0.45130661129951477,0.33724315226078033,400,0.9287251532077789,1.18302583694458,1.492938506603241,1.2071300764381885 +fresh-en-1,6,7,Q,weak_half,200,0.05128076784312725,0.07628608494997025,0.10917450711131095,0.07951844351366162,200,0.12904456555843355,0.19258302450180054,0.27663519382476803,0.19876150522381067 +fresh-en-1,6,7,Q,strong_half,200,0.2549640566110611,0.3253883570432663,0.4048720717430115,0.3263586768507957,200,0.5696114897727966,0.7096748352050781,0.9142048239707947,0.7220547769963741 +fresh-en-1,6,7,Q,random_half,200,0.11891190707683565,0.21659420430660248,0.3100777506828308,0.21527125995606183,200,0.26634009778499607,0.4506807029247284,0.6238959491252899,0.4540836029499769 +fresh-en-1,6,7,Q,all_heads,200,0.27923005223274233,0.3437023311853409,0.43859428763389585,0.3519037220627069,200,0.7047945380210876,0.8954973220825195,1.1695378422737122,0.9208162814378739 +fresh-en-1,6,7,K,weak_half,200,0.06375160813331604,0.08549407869577408,0.1117132306098938,0.08756253890693187,200,0.1543627679347992,0.20319661498069763,0.2827466785907745,0.2140925309807062 +fresh-en-1,6,7,K,strong_half,200,0.23707759231328965,0.30534569919109344,0.3755836129188537,0.3023488150537014,200,0.6287707090377808,0.7403939962387085,0.913817775249481,0.7624383002519608 +fresh-en-1,6,7,K,random_half,200,0.13201211541891097,0.2057555615901947,0.30441559553146363,0.21490271095186472,200,0.3190527677536011,0.4636679142713547,0.6615333616733551,0.4803842671215534 +fresh-en-1,6,7,K,all_heads,200,0.2558145970106125,0.32501567900180817,0.392759883403778,0.32763745248317716,200,0.794017106294632,0.9337106943130493,1.233895969390869,0.9765308284759522 +fresh-en-1,6,7,V,weak_half,200,0.07718948647379875,0.10654489696025848,0.1478450149297714,0.11025683511048555,200,0.20251110345125198,0.26464010775089264,0.3415085256099699,0.2727371209859848 +fresh-en-1,6,7,V,strong_half,200,0.3138410747051239,0.36240267753601074,0.4246575444936752,0.3697207769751549,200,0.7444261252880097,0.8773817420005798,1.0556872963905335,0.8989229846000671 +fresh-en-1,6,7,V,random_half,200,0.15545094162225723,0.2713935673236847,0.3420503556728363,0.2607215612381697,200,0.3870944380760193,0.5961516797542572,0.7767399609088897,0.5837349139153958 +fresh-en-1,6,7,V,all_heads,200,0.3441262274980545,0.39463381469249725,0.45924242436885826,0.40046052515506747,200,0.9884055316448213,1.1532039046287537,1.3873046875,1.171660108566284 +fresh-en-1,6,7,QKV,weak_half,200,0.07167162299156189,0.09132227301597595,0.12197402492165564,0.09411668732762336,200,0.180430169403553,0.22500503063201904,0.3031379282474518,0.23359243541955949 +fresh-en-1,6,7,QKV,strong_half,200,0.2856448620557785,0.3358517289161682,0.3882050573825836,0.33560425505042074,200,0.6784608483314514,0.80901038646698,0.9301543295383453,0.8072568848729134 +fresh-en-1,6,7,QKV,random_half,200,0.14224677085876464,0.23856479674577713,0.31582183241844175,0.23285716373473406,200,0.33532056510448455,0.51478311419487,0.6882146716117858,0.5151127846539021 +fresh-en-1,6,7,QKV,all_heads,200,0.31050409078598024,0.3602447658777237,0.4287660628557205,0.3631242674589157,200,0.8712185442447662,1.0402823686599731,1.223043441772461,1.0408493188023566 +fresh-en-1,6,7,JS,weak_half,200,0.0005767283611930907,0.002306645968928933,0.006926028523594141,0.003228328956574842,200,0.0010123061714693905,0.0032820418709889054,0.010579498205333947,0.004567198256481788 +fresh-en-1,6,7,JS,strong_half,200,0.021630941890180114,0.05776324309408665,0.09664976075291634,0.05778439623769373,200,0.01916338000446558,0.03676723502576351,0.060887708514928814,0.03901878243777901 +fresh-en-1,6,7,JS,random_half,200,0.008028769306838513,0.02803694549947977,0.06478861719369887,0.03246910425077658,200,0.005649351654574276,0.0184744568541646,0.04069157540798187,0.022181902052252553 +fresh-en-1,6,7,JS,all_heads,200,0.027046055532991886,0.06774412468075752,0.12803402245044707,0.0713799251220189,200,0.021029950678348543,0.04124654084444046,0.06721432581543922,0.043585980976931754 +fresh-en-1,6,7,sqrtJS,weak_half,200,0.024015053175389767,0.04802754335105419,0.08322266712784766,0.05175052645150572,200,0.07721144780516624,0.13718634843826294,0.23780937194824217,0.14866872111335397 +fresh-en-1,6,7,sqrtJS,strong_half,200,0.1470720648765564,0.24033979326486588,0.3108854293823242,0.23197421249002217,200,0.3262896716594696,0.4519944489002228,0.5877219140529633,0.45604881301522254 +fresh-en-1,6,7,sqrtJS,random_half,200,0.08960293009877206,0.16744210571050644,0.2545344918966293,0.16956594178453088,200,0.15618430227041244,0.30957549810409546,0.4402832120656967,0.3049086922779679 +fresh-en-1,6,7,sqrtJS,all_heads,200,0.164456282556057,0.2602769732475281,0.35781576633453366,0.25652276471257207,200,0.4238251566886902,0.6078069806098938,0.7762976288795471,0.6047175326943397 +fresh-en-1,7,8,Q,weak_half,400,0.11666576936841011,0.18708951026201248,0.4462867915630342,0.22673103969544173,400,0.2629485040903091,0.43455736339092255,0.9290118932724001,0.5148497210070491 +fresh-en-1,7,8,Q,strong_half,400,0.1838044673204422,0.27620406448841095,0.5339802980422974,0.33409543326124547,400,0.6578884661197663,1.1185152530670166,2.071656632423401,1.2554998660832644 +fresh-en-1,7,8,Q,random_half,400,0.1832245647907257,0.3176252841949463,0.5847894847393036,0.36109491316601633,400,0.44735789895057676,0.7670342326164246,1.5392090678215034,0.8833914407342672 +fresh-en-1,7,8,Q,all_heads,400,0.20293381810188296,0.3030116409063339,0.4967107951641084,0.338714808896184,400,0.9464879035949707,1.5582776069641113,2.910705852508545,1.7703495898842811 +fresh-en-1,7,8,K,weak_half,400,0.04877699948847295,0.15636562556028366,0.28426038026809713,0.17119474638719112,400,0.10181166678667068,0.3650897294282913,0.6156059920787812,0.39234611282590776 +fresh-en-1,7,8,K,strong_half,400,0.06368868723511696,0.23224427551031113,0.3737810224294664,0.24778394591994585,400,0.19827445298433308,0.9402804374694824,1.5164543509483337,0.9582332827523351 +fresh-en-1,7,8,K,random_half,400,0.10885747820138934,0.28821025788784027,0.5132717192173005,0.31034028535708785,400,0.14788009375333785,0.6379528343677521,1.1148837447166446,0.6741284572705627 +fresh-en-1,7,8,K,all_heads,400,0.07901620268821717,0.2390788048505783,0.39037383198738107,0.24987407720182092,400,0.2958777844905853,1.3316857814788818,2.1302282094955443,1.3505793938599526 +fresh-en-1,7,8,V,weak_half,400,0.07015151754021645,0.1445031240582466,0.25302229225635536,0.155175886079669,400,0.15762090384960176,0.29943349957466125,0.5295086920261384,0.3255165648832917 +fresh-en-1,7,8,V,strong_half,400,0.09915875867009163,0.20052561163902283,0.42266151309013367,0.23802040223032236,400,0.4298040121793747,0.9570785164833069,1.4861202359199523,0.9517359367012977 +fresh-en-1,7,8,V,random_half,400,0.11547407731413842,0.2520938515663147,0.5206403076648712,0.2956710677687079,400,0.2696614354848862,0.5846494138240814,1.0374620199203493,0.6340580058842897 +fresh-en-1,7,8,V,all_heads,400,0.10983676463365556,0.2098207026720047,0.37000077962875366,0.2245040371734649,400,0.612331521511078,1.2732290625572205,1.9485007524490359,1.2772525019943715 +fresh-en-1,7,8,QKV,weak_half,400,0.10745034292340279,0.17351055890321732,0.3211316376924515,0.19420080045238136,400,0.24533196687698364,0.38880655169487,0.71583930850029,0.4380151612497866 +fresh-en-1,7,8,QKV,strong_half,400,0.16863008141517638,0.2581538110971451,0.4188044428825379,0.2906453088670969,400,0.6563140273094177,1.108443558216095,1.6533756017684937,1.1247473461925983 +fresh-en-1,7,8,QKV,random_half,400,0.1902980908751488,0.31846967339515686,0.5253196179866791,0.3418864057213068,400,0.4052244603633881,0.7023851275444031,1.246024227142334,0.7774851971492172 +fresh-en-1,7,8,QKV,all_heads,400,0.18922706246376036,0.2694920152425766,0.397405269742012,0.2867595075443387,400,0.9311182260513305,1.4962615370750427,2.256703758239746,1.562762507647276 +fresh-en-1,8,9,Q,weak_half,400,0.06878470852971078,0.1017897017300129,0.14617488533258438,0.10487977945245802,400,0.1924812138080597,0.2792234420776367,0.4047867327928543,0.2910245754569769 +fresh-en-1,8,9,Q,strong_half,400,0.1340976133942604,0.2116369754076004,0.32891652882099154,0.22591229412704705,400,0.4058663904666901,0.6112732589244843,0.8379398822784424,0.6209514751285314 +fresh-en-1,8,9,Q,random_half,400,0.1034817323088646,0.1588492915034294,0.24468427747488022,0.16963840197771787,400,0.29587899446487426,0.4395989626646042,0.6019369065761567,0.44805550411343575 +fresh-en-1,8,9,Q,all_heads,400,0.1576593577861786,0.23841630667448044,0.36579652726650247,0.25367012023925783,400,0.6070104241371155,0.9067568182945251,1.199902093410492,0.9119760500639678 +fresh-en-1,8,9,K,weak_half,400,0.06934267282485962,0.10631198436021805,0.1705820307135582,0.114657495864667,400,0.18625243604183198,0.27490462362766266,0.40114199817180635,0.2862045105919242 +fresh-en-1,8,9,K,strong_half,400,0.14087906181812287,0.20201688259840012,0.3027588903903963,0.21435076002031564,400,0.42267473638057707,0.5850375294685364,0.7984868288040162,0.6034654051810503 +fresh-en-1,8,9,K,random_half,400,0.10777912065386773,0.16583699733018875,0.24751861542463305,0.1735384438559413,400,0.2834301233291626,0.4204404950141907,0.6028708517551422,0.43708613246679306 +fresh-en-1,8,9,K,all_heads,400,0.15370374470949175,0.2266736999154091,0.3430075347423554,0.23919859366491436,400,0.6323070466518402,0.8548831939697266,1.177220094203949,0.8896699153631925 +fresh-en-1,8,9,V,weak_half,400,0.04608457311987877,0.09232772141695023,0.17202633768320083,0.10345667429734022,400,0.13285709023475648,0.2389344871044159,0.40231205821037297,0.2556125915981829 +fresh-en-1,8,9,V,strong_half,400,0.09106388911604882,0.17171286046504974,0.31563041508197787,0.1944186894223094,400,0.29302759766578673,0.5047154426574707,0.8254227697849275,0.5405824149772525 +fresh-en-1,8,9,V,random_half,400,0.06837232634425164,0.13533063977956772,0.24439083933830266,0.15157331094145776,400,0.20008659213781357,0.3644304424524307,0.6061841666698456,0.3956356716528535 +fresh-en-1,8,9,V,all_heads,400,0.0997017577290535,0.19483598321676254,0.35056326985359193,0.21884362265467644,400,0.42908117473125457,0.751499354839325,1.2147094368934632,0.7961950064450503 +fresh-en-1,8,9,QKV,weak_half,400,0.07268035933375358,0.10562345013022423,0.15491692274808885,0.1112955713737756,400,0.20027262121438982,0.2830720990896225,0.3993861436843872,0.29051720298826694 +fresh-en-1,8,9,QKV,strong_half,400,0.15391260832548143,0.2130080610513687,0.29623655676841737,0.22042813377454876,400,0.47474442720413207,0.6070356369018555,0.781455671787262,0.6238279248028994 +fresh-en-1,8,9,QKV,random_half,400,0.11837809309363365,0.16439248621463776,0.23404497057199483,0.17170253332704305,400,0.3163149207830429,0.4390338808298111,0.586566185951233,0.45098667826503513 +fresh-en-1,8,9,QKV,all_heads,400,0.17159610986709595,0.23495026677846909,0.3334542989730835,0.24674282263964414,400,0.6932302713394165,0.8939938843250275,1.1676303625106812,0.9143451265990734 +fresh-en-1,9,10,Q,weak_half,400,0.06094711944460869,0.10585768893361092,0.18654251843690872,0.11567866146564483,400,0.16621108651161196,0.28902509808540344,0.4845675349235535,0.3115142738074064 +fresh-en-1,9,10,Q,strong_half,400,0.13207966536283494,0.22750648111104965,0.39289309382438664,0.24421696905046703,400,0.3954143702983856,0.6493356823921204,1.0021042943000793,0.6812256198376417 +fresh-en-1,9,10,Q,random_half,400,0.10025337263941765,0.165534108877182,0.28697486221790314,0.18303215925581753,400,0.281519678235054,0.4585331082344055,0.7408766329288483,0.49320222694426774 +fresh-en-1,9,10,Q,all_heads,400,0.15048966854810714,0.26013249158859253,0.4382429331541062,0.27463290037587285,400,0.5911897838115692,0.9714177846908569,1.4385213375091555,0.992739894092083 +fresh-en-1,9,10,K,weak_half,400,0.06983374580740928,0.11050503700971603,0.18521395772695542,0.119023098340258,400,0.17386156171560288,0.27945245802402496,0.42727637290954595,0.2880118969641626 +fresh-en-1,9,10,K,strong_half,400,0.12892614752054216,0.20186910033226013,0.30386801064014435,0.21343705765902996,400,0.39015957713127136,0.5710758566856384,0.7946619153022767,0.5831236628443003 +fresh-en-1,9,10,K,random_half,400,0.10964433699846268,0.16358662396669388,0.2563591420650483,0.17592061592265965,400,0.2803747922182083,0.41787467896938324,0.601813954114914,0.4318814205750823 +fresh-en-1,9,10,K,all_heads,400,0.15055901408195496,0.22853782027959824,0.3547593533992768,0.24282987158745528,400,0.5946242630481721,0.8478443622589111,1.1840943932533263,0.8711355630308389 +fresh-en-1,9,10,V,weak_half,400,0.049820014461874965,0.0957585982978344,0.18845382630825047,0.10882406448945403,400,0.13858646005392075,0.23517920821905136,0.42064121365547197,0.2608477630652487 +fresh-en-1,9,10,V,strong_half,400,0.09640422835946083,0.15963114798069,0.3254160404205323,0.18729992019943892,400,0.30360322296619413,0.4640680104494095,0.8102843344211582,0.522186496220529 +fresh-en-1,9,10,V,random_half,400,0.07403680831193923,0.12740062177181244,0.2536500900983811,0.15287961770780384,400,0.2124659463763237,0.34833425283432007,0.6163441002368928,0.38827623162418606 +fresh-en-1,9,10,V,all_heads,400,0.11297901272773743,0.19091308116912842,0.36864126026630406,0.21979761159978806,400,0.4665547549724579,0.6942921578884125,1.19773188829422,0.783034263253212 +fresh-en-1,9,10,QKV,weak_half,400,0.06883239522576333,0.11383510380983353,0.17397884428501129,0.11860841455869377,400,0.17947925329208375,0.2906233072280884,0.4189550638198853,0.3003467996045947 +fresh-en-1,9,10,QKV,strong_half,400,0.14540800154209138,0.21212786436080933,0.32404509186744695,0.22453425219282508,400,0.4376334309577942,0.6155771017074585,0.8541621088981629,0.6340144545584917 +fresh-en-1,9,10,QKV,random_half,400,0.11538342982530594,0.1700264737010002,0.24538346827030194,0.1781065428815782,400,0.3064898431301117,0.4569401144981384,0.647545462846756,0.46353640422225 +fresh-en-1,9,10,QKV,all_heads,400,0.17007683366537094,0.24171700328588486,0.36055758595466614,0.25575132431462405,400,0.6377313256263734,0.9152962565422058,1.2620975852012637,0.9343612555414438 +fresh-en-1,10,11,Q,weak_half,200,0.06944099664688111,0.09974412620067596,0.1316825270652771,0.10007419964298606,200,0.18252173364162447,0.2599073648452759,0.3511184364557266,0.2633637668192387 +fresh-en-1,10,11,Q,strong_half,200,0.1699372336268425,0.22559165209531784,0.30074552595615384,0.231473029255867,200,0.4364379048347473,0.5961241722106934,0.7720701932907105,0.6011680337786675 +fresh-en-1,10,11,Q,random_half,200,0.11976733207702636,0.17453531920909882,0.2334069699048996,0.1762715058028698,200,0.2938146501779556,0.4288535863161087,0.5812847852706909,0.43108469240367414 +fresh-en-1,10,11,Q,all_heads,200,0.19902106374502185,0.2585810273885727,0.3368082493543625,0.26147273384034636,200,0.623351377248764,0.8477539420127869,1.131349778175354,0.8645317968726158 +fresh-en-1,10,11,K,weak_half,200,0.05753650888800621,0.08063225820660591,0.11064159125089645,0.08343798661604523,200,0.14854126870632173,0.2100462019443512,0.27082476019859314,0.2124735986441374 +fresh-en-1,10,11,K,strong_half,200,0.18665273636579513,0.266862690448761,0.35107639133930196,0.2716717680543661,200,0.48777368366718293,0.630791425704956,0.7756236135959625,0.6299147251248359 +fresh-en-1,10,11,K,random_half,200,0.11451399922370911,0.17006069421768188,0.2834821879863739,0.1885000267997384,200,0.2704488217830658,0.4074474275112152,0.5655113577842712,0.4160751291364431 +fresh-en-1,10,11,K,all_heads,200,0.20212250351905822,0.2926028370857239,0.3856657564640045,0.2924680233001709,200,0.6800970137119293,0.8471084237098694,0.9841707348823545,0.8423883187770843 +fresh-en-1,10,11,V,weak_half,200,0.07907667607069016,0.1134326122701168,0.15122054517269135,0.11284223183989525,200,0.20612292140722277,0.29504191875457764,0.4048506021499634,0.2969295750558376 +fresh-en-1,10,11,V,strong_half,200,0.18174155056476593,0.2496490329504013,0.3162019371986389,0.24678777687251569,200,0.5428029775619507,0.6820696592330933,0.802109831571579,0.6706108778715134 +fresh-en-1,10,11,V,random_half,200,0.13125766068696976,0.18281591683626175,0.2498991474509239,0.18792357690632344,200,0.3371214807033539,0.47089941799640656,0.62029110789299,0.4780918924510479 +fresh-en-1,10,11,V,all_heads,200,0.21311977058649065,0.27266694605350494,0.34110399484634396,0.27386617958545684,200,0.7686313748359681,0.9857306182384491,1.1916023731231689,0.9675404632091522 +fresh-en-1,10,11,QKV,weak_half,200,0.06905001550912858,0.10094969347119331,0.12833732813596727,0.09993800090625882,200,0.18383744955062867,0.2632672041654587,0.34155472517013546,0.2626323198527098 +fresh-en-1,10,11,QKV,strong_half,200,0.2015552669763565,0.252907931804657,0.3102490305900574,0.2531704943627119,200,0.5313649356365204,0.6640969514846802,0.7787599325180053,0.6562527805566788 +fresh-en-1,10,11,QKV,random_half,200,0.128659687936306,0.1878657117486,0.24712180495262145,0.18670625861734153,200,0.3133641630411148,0.4526396840810776,0.5957495152950287,0.4546309668570757 +fresh-en-1,10,11,QKV,all_heads,200,0.2186402276158333,0.2814401835203171,0.3346459418535232,0.2786870900541544,200,0.7310712456703186,0.924795001745224,1.1009344816207887,0.918885104060173 +fresh-en-1,10,11,JS,weak_half,200,0.0006487292353995145,0.002476100460626185,0.007742555718868969,0.0035485731244261844,200,0.0011850937851704659,0.003247117158025503,0.007588309375569224,0.0038854427193291486 +fresh-en-1,10,11,JS,strong_half,200,0.011076398845762015,0.030100420117378235,0.06291758120059968,0.034063537010224534,200,0.010585208982229232,0.0233779763802886,0.04796599671244621,0.027230191302951425 +fresh-en-1,10,11,JS,random_half,200,0.0075511276256293065,0.016958865337073803,0.04382962100207806,0.02106688379950356,200,0.004978407686576247,0.012179754674434662,0.028055685944855212,0.015364673343137838 +fresh-en-1,10,11,JS,all_heads,200,0.014377229474484922,0.03360266052186489,0.07278357818722724,0.040764510629232975,200,0.012129753734916449,0.027565681375563145,0.053156211599707605,0.031115633946610613 +fresh-en-1,10,11,sqrtJS,weak_half,200,0.025470119901001453,0.04975937679409981,0.08798861578106879,0.053749845540151,200,0.09070465341210365,0.15358279645442963,0.22870372980833054,0.1554449757374823 +fresh-en-1,10,11,sqrtJS,strong_half,200,0.10524024218320847,0.17349450290203094,0.25083377957344055,0.1755852117575705,200,0.25386994481086733,0.3907646983861923,0.514481920003891,0.3872256969660521 +fresh-en-1,10,11,sqrtJS,random_half,200,0.08689491674304009,0.13022612035274506,0.20935525000095367,0.13791918747127055,200,0.16560839712619782,0.2626743018627167,0.4054365575313568,0.2716565002501011 +fresh-en-1,10,11,sqrtJS,all_heads,200,0.11990358531475068,0.1833082139492035,0.2697817355394363,0.19265078946948053,200,0.3480386197566986,0.5435992777347565,0.7279269158840179,0.5426706725358963 +fresh-en-1,11,12,Q,weak_half,400,0.1238834410905838,0.2069108635187149,0.36830435991287236,0.22872868400067092,400,0.24347191601991655,0.3482319116592407,0.5344525635242462,0.3711732878908515 +fresh-en-1,11,12,Q,strong_half,400,0.1404658243060112,0.2178381159901619,0.39339359402656554,0.24297691743820907,400,0.47751598060131073,0.7311754822731018,1.2181585907936097,0.7982833039015531 +fresh-en-1,11,12,Q,random_half,400,0.12326943576335908,0.2052604854106903,0.4111480683088303,0.24412159588187932,400,0.32472758293151854,0.5374113619327545,0.9631309926509858,0.5961006556078792 +fresh-en-1,11,12,Q,all_heads,400,0.11672314479947091,0.17196805030107498,0.27572976350784306,0.1832105772010982,400,0.7567769765853882,1.0795705318450928,1.7580996870994572,1.1694565898925067 +fresh-en-1,11,12,K,weak_half,400,0.10059513375163079,0.17529841512441635,0.3090489774942399,0.19310834203846752,400,0.17775298655033112,0.27525240182876587,0.4497947871685029,0.29766172748059033 +fresh-en-1,11,12,K,strong_half,400,0.10450904220342637,0.16252351552248,0.2757287234067917,0.18061625469475984,400,0.33033922910690305,0.5657155215740204,0.9709789872169496,0.6175841840729117 +fresh-en-1,11,12,K,random_half,400,0.10739448666572571,0.18538132309913635,0.3485639333724976,0.22060620564967393,400,0.25274287462234496,0.411623552441597,0.7466988205909729,0.4620770391821861 +fresh-en-1,11,12,K,all_heads,400,0.10506204515695572,0.15816397964954376,0.2476702481508256,0.17015537584200502,400,0.5281048655509949,0.8310451805591583,1.4153220534324649,0.9152459093928337 +fresh-en-1,11,12,V,weak_half,400,0.07375210300087928,0.14814569056034088,0.268029722571373,0.16095607691444458,400,0.14607070088386537,0.2344798892736435,0.43328196406364444,0.2665331773087382 +fresh-en-1,11,12,V,strong_half,400,0.07723352313041687,0.13354657590389252,0.2692253470420839,0.15651939538307488,400,0.25895796716213226,0.4647410809993744,0.9689955949783325,0.555792277008295 +fresh-en-1,11,12,V,random_half,400,0.07810793444514275,0.14845558255910873,0.32709783613681803,0.1883373015653342,400,0.19380923211574555,0.3508721739053726,0.6997187316417696,0.41212878055870533 +fresh-en-1,11,12,V,all_heads,400,0.08194395154714584,0.14404284209012985,0.25525235235691074,0.15919808047823608,400,0.4096657246351242,0.7090068459510803,1.372148954868317,0.822325454801321 +fresh-en-1,11,12,QKV,weak_half,400,0.12007122412323952,0.19698374718427658,0.3041389286518097,0.20461925929412245,400,0.22870806306600572,0.31007519364356995,0.46732245683670043,0.32787769224494695 +fresh-en-1,11,12,QKV,strong_half,400,0.1343732312321663,0.18974439799785614,0.29420072436332706,0.20492610456421972,400,0.42669931054115295,0.6640896797180176,1.0239916801452638,0.6980599980801344 +fresh-en-1,11,12,QKV,random_half,400,0.12504710182547568,0.20223911851644516,0.35009779036045074,0.23108940072357653,400,0.3058811962604523,0.4742127060890198,0.8053206801414491,0.519703901745379 +fresh-en-1,11,12,QKV,all_heads,400,0.12374101132154465,0.16487182676792145,0.24052426815032962,0.17661182712763548,400,0.6695722460746766,0.9737617075443268,1.4699242115020752,1.0259376887232066 +fresh-en-1,12,13,Q,weak_half,400,0.04199954308569431,0.06774314120411873,0.10672791749238969,0.0711545211263001,400,0.12076444029808045,0.18298646807670593,0.2665613472461701,0.18831987583078444 +fresh-en-1,12,13,Q,strong_half,400,0.09062614068388938,0.14761626720428467,0.2523013204336167,0.1606011653225869,400,0.2737719863653183,0.4510657787322998,0.7102379858493806,0.47010160353034736 +fresh-en-1,12,13,Q,random_half,400,0.06791080385446549,0.11098757013678551,0.18878157138824467,0.12181354452855885,400,0.19356008768081664,0.3096689283847809,0.48502295613288887,0.3263991795666516 +fresh-en-1,12,13,Q,all_heads,400,0.10142687857151031,0.16127675026655197,0.2634800791740418,0.17517426386475562,400,0.40015879869461063,0.6367415189743042,0.9498837471008302,0.658421477265656 +fresh-en-1,12,13,K,weak_half,400,0.04000897668302059,0.0662442147731781,0.10698441714048386,0.07137605293188244,400,0.10994575917720795,0.1685384139418602,0.2603339970111847,0.17883335843682288 +fresh-en-1,12,13,K,strong_half,400,0.08951610177755356,0.12884393334388733,0.19378753304481508,0.13636655246838927,400,0.2655521184206009,0.3705669790506363,0.5334886610507966,0.3865734089538455 +fresh-en-1,12,13,K,random_half,400,0.06977591663599016,0.10822867229580879,0.16802441477775576,0.11436999661847949,400,0.18019572049379348,0.2777932435274124,0.39742228090763093,0.28388734923675657 +fresh-en-1,12,13,K,all_heads,400,0.0973131999373436,0.1412707194685936,0.21128804236650467,0.1499322714470327,400,0.3927572250366211,0.5375449061393738,0.7774453818798066,0.5654067690297961 +fresh-en-1,12,13,V,weak_half,400,0.03237935602664948,0.05586152896285057,0.09647395685315133,0.062046736229676755,400,0.09038065373897552,0.13788507878780365,0.22541089355945587,0.15078939713537692 +fresh-en-1,12,13,V,strong_half,400,0.06915062367916107,0.1109115295112133,0.1759899526834488,0.11871332617476582,400,0.24378553926944732,0.3483941853046417,0.48023294508457187,0.3595046537742019 +fresh-en-1,12,13,V,random_half,400,0.05359541215002537,0.09506300836801529,0.159262253344059,0.10142002778593451,400,0.16232227236032487,0.24565041810274124,0.3550059884786606,0.25703549172729256 +fresh-en-1,12,13,V,all_heads,400,0.07847891598939896,0.11983677372336388,0.18962698429822927,0.12814533180557192,400,0.3575849711894989,0.49409888684749603,0.699405062198639,0.5102940502017737 +fresh-en-1,12,13,QKV,weak_half,400,0.04619770012795925,0.06748224794864655,0.1004384994506836,0.07057117450982332,400,0.12104464918375016,0.17497044801712036,0.24026590287685398,0.17977515794336796 +fresh-en-1,12,13,QKV,strong_half,400,0.10085328444838525,0.13891717791557312,0.19544970989227295,0.14492867968976497,400,0.31298905313014985,0.42447489500045776,0.5549153685569764,0.42992491722106935 +fresh-en-1,12,13,QKV,random_half,400,0.08043641299009323,0.1137194074690342,0.1631809949874878,0.1174005822557956,400,0.20837896764278413,0.29573529958724976,0.40936724841594707,0.30440974853932856 +fresh-en-1,12,13,QKV,all_heads,400,0.11095675975084304,0.15141302347183228,0.2091833993792534,0.15752761216834188,400,0.4490968585014343,0.6040551364421844,0.7720139563083649,0.6097000724822282 +fresh-en-1,13,14,Q,weak_half,400,0.0333906464278698,0.05858255550265312,0.10196247473359109,0.06311847380362451,400,0.09302718043327332,0.15997984260320663,0.2642755776643753,0.1702065001707524 +fresh-en-1,13,14,Q,strong_half,400,0.09204253703355789,0.15910228341817856,0.26018022298812865,0.16915235366672277,400,0.27269009649753573,0.4473327398300171,0.6860464870929719,0.46286991441622377 +fresh-en-1,13,14,Q,random_half,400,0.06082731001079082,0.11773044615983963,0.20547893792390826,0.1256006505060941,400,0.17466874718666078,0.29920512437820435,0.5050004959106447,0.31935379272326825 +fresh-en-1,13,14,Q,all_heads,400,0.1011314556002617,0.1709803193807602,0.2773810833692551,0.18358654756098985,400,0.3858739286661148,0.6040984690189362,0.9251342713832856,0.6330764130875468 +fresh-en-1,13,14,K,weak_half,400,0.040776994451880454,0.06571542471647263,0.11363649964332584,0.07244727050419897,400,0.1067556381225586,0.1667572408914566,0.26869713962078096,0.17781931541860105 +fresh-en-1,13,14,K,strong_half,400,0.10767457932233811,0.16511210054159164,0.24289021342992784,0.17294135492295026,400,0.3147138386964798,0.45152726769447327,0.6594492793083191,0.47442216757684946 +fresh-en-1,13,14,K,random_half,400,0.08041624277830124,0.1300177499651909,0.22916551381349565,0.14435280523262917,400,0.1976301357150078,0.31191785633563995,0.4727044641971589,0.32789318487048147 +fresh-en-1,13,14,K,all_heads,400,0.11809397861361505,0.17358045279979706,0.2585149049758912,0.18222347900271416,400,0.431104251742363,0.6153574883937836,0.931251049041748,0.6522414813935756 +fresh-en-1,13,14,V,weak_half,400,0.027668917924165728,0.04883405566215515,0.10824857205152516,0.06140576663659886,400,0.07761971652507782,0.12573891878128052,0.25847788453102116,0.15221538756042718 +fresh-en-1,13,14,V,strong_half,400,0.06748246252536774,0.12791181355714798,0.2509697258472443,0.14929755680263043,400,0.21045585572719575,0.3495970815420151,0.6511614680290223,0.4036193377897143 +fresh-en-1,13,14,V,random_half,400,0.05010525286197663,0.09721117839217186,0.2035676896572114,0.11763474846724421,400,0.13613949716091156,0.242456316947937,0.4468719005584717,0.28071628173813223 +fresh-en-1,13,14,V,all_heads,400,0.07606047689914705,0.13386619091033936,0.26358460187911986,0.15910425290465355,400,0.29006458818912506,0.48444347083568573,0.9060238480567933,0.5558347230032087 +fresh-en-1,13,14,QKV,weak_half,400,0.04081669710576535,0.06304764375090599,0.10695586055517199,0.06867555658333004,400,0.10514951199293136,0.16336672753095627,0.2553859144449234,0.17548505634069442 +fresh-en-1,13,14,QKV,strong_half,400,0.11359325945377349,0.16603676974773407,0.23806603252887726,0.17177847370505334,400,0.32835593521595,0.4530118554830551,0.6462366938591003,0.4755962960794568 +fresh-en-1,13,14,QKV,random_half,400,0.07887387722730636,0.12440208718180656,0.2024524182081223,0.13602644935250283,400,0.20758252292871476,0.30556483566761017,0.4701956987380982,0.3282794947549701 +fresh-en-1,13,14,QKV,all_heads,400,0.12154724448919296,0.17428946495056152,0.2550203919410706,0.18301012860611082,400,0.4466926246881485,0.6231760382652283,0.8954059839248659,0.6510813497751952 +fresh-en-1,14,15,Q,weak_half,200,0.05972525961697102,0.08768754824995995,0.12654390782117841,0.09146249745041132,200,0.16171305477619172,0.23234986513853073,0.3349693536758423,0.2418448817357421 +fresh-en-1,14,15,Q,strong_half,200,0.21042995005846024,0.26153530180454254,0.324012291431427,0.26548121698200705,200,0.49634958505630494,0.6425824761390686,0.8015037953853607,0.6476942497491837 +fresh-en-1,14,15,Q,random_half,200,0.12157834991812706,0.18700086325407028,0.2497889772057533,0.1866412503272295,200,0.2847464978694916,0.45379510521888733,0.5851304411888123,0.4428778136521578 +fresh-en-1,14,15,Q,all_heads,200,0.21908943206071854,0.2742941975593567,0.33914255499839785,0.2768461449444294,200,0.6563409686088562,0.8885973691940308,1.1189436435699462,0.8895391279459 +fresh-en-1,14,15,K,weak_half,200,0.03400024399161339,0.05253284424543381,0.07024108618497849,0.051974865505471826,200,0.07971110939979553,0.1251831203699112,0.17426325380802155,0.1268891525827348 +fresh-en-1,14,15,K,strong_half,200,0.14096638560295105,0.22103718668222427,0.34185391664505005,0.23046200100332498,200,0.3435581922531128,0.5238255262374878,0.7481223046779631,0.5397826400399208 +fresh-en-1,14,15,K,random_half,200,0.08331794515252114,0.14197427779436111,0.2737092196941376,0.1595736616104841,200,0.19068296402692794,0.3037271052598953,0.5456008434295654,0.3334548348933458 +fresh-en-1,14,15,K,all_heads,200,0.1498803660273552,0.2287801057100296,0.3448011815547943,0.2389882105588913,200,0.44229177534580233,0.6692723035812378,0.898008644580841,0.6666717946529388 +fresh-en-1,14,15,V,weak_half,200,0.0313275046646595,0.04746314883232117,0.07551179230213165,0.05051631234586239,200,0.08001820892095567,0.11556577682495117,0.1866932451725006,0.12669924590736628 +fresh-en-1,14,15,V,strong_half,200,0.11879860609769821,0.17190982401371002,0.30756547749042507,0.19820406567305326,200,0.2921076089143753,0.4412684142589569,0.6660638451576233,0.46328026719391346 +fresh-en-1,14,15,V,random_half,200,0.07624801844358445,0.12988001853227615,0.2257589653134346,0.1404957754164934,200,0.15936479717493057,0.2751575708389282,0.45192498862743374,0.2943941374868155 +fresh-en-1,14,15,V,all_heads,200,0.1338060662150383,0.18094080686569214,0.3125566303730011,0.20830298632383346,200,0.38464597165584563,0.5482084155082703,0.837924814224243,0.5899795150756836 +fresh-en-1,14,15,QKV,weak_half,200,0.04821562208235264,0.06717341020703316,0.09222997650504113,0.06822513587772847,200,0.12617674916982652,0.17499307543039322,0.24576923996210095,0.17911065086722375 +fresh-en-1,14,15,QKV,strong_half,200,0.2020306780934334,0.23559602349996567,0.27765149176120757,0.2398426128178835,200,0.4935259848833084,0.5986289978027344,0.7088593661785125,0.5998339432477952 +fresh-en-1,14,15,QKV,random_half,200,0.11645718067884446,0.16869307309389114,0.22687322795391082,0.16977287262678145,200,0.26859555542469027,0.3867724984884262,0.5149533152580261,0.38845198668539527 +fresh-en-1,14,15,QKV,all_heads,200,0.21109243035316466,0.25006889551877975,0.2848523288965225,0.2495175714790821,200,0.6223420441150666,0.7783544957637787,0.9231582403182983,0.7789445948600769 +fresh-en-1,14,15,JS,weak_half,200,0.0007763063360471279,0.002509985235519707,0.00766885820776224,0.003732908903621137,200,0.0011543412460014225,0.0032102898694574833,0.008047807589173317,0.0040331551556300835 +fresh-en-1,14,15,JS,strong_half,200,0.011302258260548115,0.024751482531428337,0.05091860070824622,0.029279520583804695,200,0.015055215358734131,0.029228592291474342,0.0523753385990858,0.032888505052542316 +fresh-en-1,14,15,JS,random_half,200,0.0051984306890517475,0.014517479110509157,0.03340637423098087,0.01769778228655923,200,0.006468428997322917,0.01561273168772459,0.033602714166045176,0.018580437625641936 +fresh-en-1,14,15,JS,all_heads,200,0.011281129065901042,0.025093545205891132,0.05286857895553111,0.029380197356222198,200,0.01800473090261221,0.033482665196061134,0.05922782830893993,0.03692166009568609 +fresh-en-1,14,15,sqrtJS,weak_half,200,0.027862273529171942,0.050099652260541916,0.08757161423563957,0.05542821571230888,200,0.0852257788181305,0.14180681854486465,0.2221267521381378,0.14699578979983927 +fresh-en-1,14,15,sqrtJS,strong_half,200,0.10631204769015312,0.15732577443122864,0.225648558139801,0.16332259381189942,200,0.3110693901777268,0.4123976528644562,0.576462596654892,0.43372884660959243 +fresh-en-1,14,15,sqrtJS,random_half,200,0.07210011184215545,0.12048682570457458,0.18277189731597898,0.12508158395066857,200,0.18808934092521667,0.27997562289237976,0.4204215109348297,0.29344766434282066 +fresh-en-1,14,15,sqrtJS,all_heads,200,0.10621153041720391,0.15840937197208405,0.22993040680885313,0.16380997877568007,200,0.44155151546001437,0.5724745690822601,0.7562368214130402,0.5807246388494969 +fresh-en-1,15,16,Q,weak_half,400,0.05115569531917572,0.11163235083222389,0.19201354086399083,0.11851625631097704,400,0.09324264898896217,0.20010404288768768,0.2961266309022904,0.20080076995305718 +fresh-en-1,15,16,Q,strong_half,400,0.07385298833251,0.16069813072681427,0.3015475064516068,0.17814022227190435,400,0.17087184935808183,0.3426547944545746,0.6329521954059601,0.3731372333317995 +fresh-en-1,15,16,Q,random_half,400,0.06568787023425103,0.14229707419872284,0.25226762294769295,0.15374847989995033,400,0.12571520805358888,0.26651036739349365,0.4767976969480516,0.2908402467984706 +fresh-en-1,15,16,Q,all_heads,400,0.1157577946782112,0.22759568691253662,0.3756236761808396,0.240536219868809,400,0.2711366653442383,0.5396303236484528,0.9237230241298676,0.573938002679497 +fresh-en-1,15,16,K,weak_half,400,0.0720119334757328,0.11476623639464378,0.1975547417998314,0.1259802564326674,400,0.12744511365890504,0.1808849647641182,0.27546829581260684,0.19104495994746684 +fresh-en-1,15,16,K,strong_half,400,0.09909193962812424,0.15460841357707977,0.2442264422774315,0.1652323190867901,400,0.21407503336668016,0.32006849348545074,0.5206866800785065,0.3470751639455557 +fresh-en-1,15,16,K,random_half,400,0.09023350849747659,0.14198295027017593,0.2503163427114487,0.15840351218357682,400,0.16206704080104828,0.23965883255004883,0.4151101619005204,0.2684745015017688 +fresh-en-1,15,16,K,all_heads,400,0.14247250109910967,0.22148402780294418,0.3355022430419922,0.23079961340874433,400,0.3425900548696518,0.500378429889679,0.7856600284576417,0.5381201263144613 +fresh-en-1,15,16,V,weak_half,400,0.06892155334353448,0.11238246038556099,0.1934076949954033,0.12435553018003702,400,0.1185330018401146,0.16938398778438568,0.2523349642753602,0.1777927975077182 +fresh-en-1,15,16,V,strong_half,400,0.09883744940161705,0.16184579581022263,0.2384017437696457,0.16994249603711067,400,0.20923011600971222,0.3066157400608063,0.47518199682235723,0.32719085227698086 +fresh-en-1,15,16,V,random_half,400,0.08600258752703667,0.1390918716788292,0.2349163308739663,0.15318310870788993,400,0.1540341153740883,0.23507018387317657,0.3749436795711517,0.2510602937079966 +fresh-en-1,15,16,V,all_heads,400,0.1690642610192299,0.22808347642421722,0.335255891084671,0.24334939943626524,400,0.32907225787639616,0.4751551002264023,0.7278320550918579,0.5049836485087872 +fresh-en-1,15,16,QKV,weak_half,400,0.07989880442619324,0.11726068705320358,0.19094224572181703,0.12669851871207358,400,0.14025417864322662,0.18742965906858444,0.2664393067359924,0.19686487335711716 +fresh-en-1,15,16,QKV,strong_half,400,0.10922941565513611,0.16662801802158356,0.24773750603199007,0.17695572426542638,400,0.24858227968215943,0.3311450481414795,0.5379368007183075,0.3661909296736121 +fresh-en-1,15,16,QKV,random_half,400,0.10014482289552688,0.1484013870358467,0.23843082487583162,0.1609893567673862,400,0.18035451024770738,0.25932882726192474,0.41887034773826604,0.28241308603435755 +fresh-en-1,15,16,QKV,all_heads,400,0.17184289246797563,0.23527216911315918,0.33110528588294985,0.2449055170454085,400,0.3923039108514786,0.5221433043479919,0.8025936484336853,0.5630558013916016 +fresh-en-1,16,17,Q,weak_half,400,0.01090065520256758,0.03213384933769703,0.06482484787702561,0.03540428388747387,400,0.029532276280224323,0.0893373042345047,0.17565270066261293,0.09621945450082421 +fresh-en-1,16,17,Q,strong_half,400,0.024700336903333668,0.08621794357895851,0.1668551594018937,0.0934323610062711,400,0.06219766028225422,0.22039325535297394,0.42419965565204626,0.23902779089286924 +fresh-en-1,16,17,Q,random_half,400,0.016319026052951813,0.062303658574819565,0.127023857831955,0.06753292427631095,400,0.04610161222517491,0.1591755449771881,0.3023368239402771,0.16779809141997248 +fresh-en-1,16,17,Q,all_heads,400,0.02626625243574381,0.09000314399600029,0.1832762107253075,0.0997350847045891,400,0.09337917640805246,0.30953696370124817,0.5907247006893158,0.33524724560789765 +fresh-en-1,16,17,K,weak_half,400,0.03309944570064545,0.05013861693441868,0.0803563952445984,0.05375516610220075,400,0.08944721966981889,0.12850934267044067,0.17739043831825257,0.13129173380322753 +fresh-en-1,16,17,K,strong_half,400,0.07427202835679055,0.11068788170814514,0.16004806458950044,0.11409356572665275,400,0.22308700978755952,0.30215004086494446,0.434687539935112,0.31780049700289964 +fresh-en-1,16,17,K,random_half,400,0.05551785156130791,0.08376788347959518,0.1244123667478562,0.08718048962764442,400,0.1439796969294548,0.2100352942943573,0.32709612846374514,0.2241826919838786 +fresh-en-1,16,17,K,all_heads,400,0.08231901973485947,0.12072939798235893,0.1722958534955979,0.1252398610673845,400,0.3151325792074204,0.43429818749427795,0.6140280783176422,0.44909223057329656 +fresh-en-1,16,17,V,weak_half,400,0.026491530612111094,0.045925892889499664,0.07523442134261132,0.04897580315358937,400,0.07256868034601212,0.11991117894649506,0.18080151528120042,0.12262617296539247 +fresh-en-1,16,17,V,strong_half,400,0.07131528109312057,0.1119205541908741,0.1869285121560097,0.12265017106197774,400,0.19829190522432327,0.30642981827259064,0.4262947112321854,0.31290188178420064 +fresh-en-1,16,17,V,random_half,400,0.049160628020763396,0.08027539029717445,0.14036452472209932,0.08883013811893761,400,0.12876918613910676,0.2056737244129181,0.3154082834720612,0.2181542399339378 +fresh-en-1,16,17,V,all_heads,400,0.07808888480067253,0.12669668346643448,0.2022019788622856,0.13496380747295916,400,0.27350446581840515,0.4227070212364197,0.5940569043159486,0.43552805356681346 +fresh-en-1,16,17,QKV,weak_half,400,0.03333875611424446,0.047355176880955696,0.0668675646185875,0.049026281065307556,400,0.0897772952914238,0.12252087518572807,0.16235407888889314,0.12462929629720748 +fresh-en-1,16,17,QKV,strong_half,400,0.0815234273672104,0.11003098264336586,0.16163104623556138,0.11585234424099326,400,0.22960261702537538,0.30487804114818573,0.41486041545867924,0.3122369610145688 +fresh-en-1,16,17,QKV,random_half,400,0.05682018473744392,0.08245310932397842,0.12116478011012079,0.08600277340039611,400,0.14840557724237444,0.20900710672140121,0.3014710545539856,0.21840926218777895 +fresh-en-1,16,17,QKV,all_heads,400,0.08897768259048462,0.12090852111577988,0.1753150299191475,0.12663230292499064,400,0.32244964838027956,0.4269644618034363,0.5670095801353455,0.43686625722795724 +fresh-en-1,17,18,Q,weak_half,400,0.009794714394956827,0.029066860675811768,0.05373833738267422,0.030948201109422372,400,0.027484984882175923,0.08109355717897415,0.14353146255016327,0.08451876983977855 +fresh-en-1,17,18,Q,strong_half,400,0.032937375083565715,0.10008666291832924,0.18823635876178746,0.10699740604031831,400,0.08021067753434183,0.2587098926305771,0.42698949575424194,0.25606562359258533 +fresh-en-1,17,18,Q,random_half,400,0.020677325874567033,0.06470144540071487,0.1366383045911789,0.07546006517484784,400,0.052433885261416444,0.1593521386384964,0.2935037583112717,0.17096915594767778 +fresh-en-1,17,18,Q,all_heads,400,0.03479687608778477,0.10972586274147034,0.19641194194555284,0.11406105661764741,400,0.1102134570479393,0.3384764492511749,0.5653740286827089,0.34058439357206227 +fresh-en-1,17,18,K,weak_half,400,0.012865278124809266,0.026060963049530983,0.042548727616667746,0.02738986423937604,400,0.03723554387688637,0.07079154253005981,0.10603573173284533,0.07265213681850582 +fresh-en-1,17,18,K,strong_half,400,0.04658172316849232,0.08491340652108192,0.13339817374944687,0.08848608897998929,400,0.11590702533721925,0.2061862200498581,0.2925162136554718,0.20790251709520816 +fresh-en-1,17,18,K,random_half,400,0.03295996338129044,0.05984056182205677,0.10925500765442848,0.06533183038467541,400,0.07415947765111923,0.1354980394244194,0.21232306957244873,0.14058071721345186 +fresh-en-1,17,18,K,all_heads,400,0.04742027968168259,0.08967533707618713,0.14682747125625611,0.09451059772633016,400,0.1528210535645485,0.28098051249980927,0.39314436912536627,0.28055465370416643 +fresh-en-1,17,18,V,weak_half,400,0.019553527235984802,0.028215929865837097,0.04272730313241482,0.02986624018987641,400,0.05096744112670422,0.07679607346653938,0.10720844268798829,0.07800233666785061 +fresh-en-1,17,18,V,strong_half,400,0.08688115552067757,0.12466416880488396,0.17578225433826447,0.12853062788024544,400,0.2175402045249939,0.2835098057985306,0.3654308021068573,0.28902593825012446 +fresh-en-1,17,18,V,random_half,400,0.04126483239233494,0.08443579822778702,0.1399387910962105,0.08815104129258543,400,0.10063198953866959,0.1810707300901413,0.27093708217144014,0.18461735697463155 +fresh-en-1,17,18,V,all_heads,400,0.09076240360736847,0.12774693965911865,0.177287957072258,0.13264115741476418,400,0.27749455571174625,0.3594724237918854,0.46194404959678653,0.36702827613800765 +fresh-en-1,17,18,QKV,weak_half,400,0.020704022236168386,0.02985592745244503,0.04230735674500465,0.030972254781518133,400,0.057423806935548785,0.07871405780315399,0.11195403635501862,0.0828141103219241 +fresh-en-1,17,18,QKV,strong_half,400,0.08144553750753403,0.11029660329222679,0.1501531928777695,0.11451834741048515,400,0.19287436604499816,0.26389819383621216,0.34331349134445194,0.26911254759877923 +fresh-en-1,17,18,QKV,random_half,400,0.04345549494028092,0.0804177075624466,0.1174861788749695,0.08128176008351147,400,0.1057484433054924,0.17332039773464203,0.25759845674037934,0.17658276157453656 +fresh-en-1,17,18,QKV,all_heads,400,0.08439640328288078,0.1163993701338768,0.1590365931391716,0.12062871979549528,400,0.25511411130428313,0.34712663292884827,0.4507107973098755,0.35192665718495847 +fresh-en-1,18,19,Q,weak_half,200,0.04607245996594429,0.057622361928224564,0.07538551613688468,0.059285122957080605,200,0.12817559242248536,0.1542971432209015,0.1979708269238472,0.1587935135513544 +fresh-en-1,18,19,Q,strong_half,200,0.10979175195097923,0.14186941832304,0.1820901572704315,0.14289242796599866,200,0.29087975025177004,0.3492428660392761,0.41580881774425504,0.3519573538005352 +fresh-en-1,18,19,Q,random_half,200,0.07439967691898346,0.10161803662776947,0.14425532519817352,0.10611185221001505,200,0.19782067239284515,0.25445254147052765,0.3200387954711914,0.25813284754753113 +fresh-en-1,18,19,Q,all_heads,200,0.12516989409923554,0.15264279395341873,0.19349252581596374,0.1554434635862708,200,0.4311848908662796,0.5070521533489227,0.6162232995033264,0.510750867575407 +fresh-en-1,18,19,K,weak_half,200,0.021145398914813995,0.029543166048824787,0.04602242074906825,0.03194696414284408,200,0.05763418450951576,0.07531525567173958,0.11713674664497375,0.08217812972143293 +fresh-en-1,18,19,K,strong_half,200,0.06190391629934311,0.09097854420542717,0.14197815954685206,0.0960666169039905,200,0.1681639850139618,0.2236045002937317,0.31033720970153805,0.23286259651184082 +fresh-en-1,18,19,K,random_half,200,0.04023327194154263,0.06545683369040489,0.1000815361738205,0.06975728743709624,200,0.09552106484770775,0.15109624713659286,0.21948905885219572,0.158261862359941 +fresh-en-1,18,19,K,all_heads,200,0.06707860752940178,0.09552209824323654,0.15241851359605788,0.10292551666498184,200,0.22670026421546938,0.3070446252822876,0.41831666231155396,0.3150407254695892 +fresh-en-1,18,19,V,weak_half,200,0.029709825292229652,0.03895639255642891,0.07712270468473431,0.04682686937972903,200,0.06536544859409332,0.08957197144627571,0.21564661264419546,0.11229447219520808 +fresh-en-1,18,19,V,strong_half,200,0.06292868405580522,0.09015366062521935,0.199218226969242,0.10598047569394112,200,0.12313150614500047,0.1776788905262947,0.4950168281793592,0.22915921442210674 +fresh-en-1,18,19,V,random_half,200,0.041984017193317416,0.06753698363900185,0.13781057298183438,0.07904955218546092,200,0.09368348568677902,0.13696441799402237,0.3379938066005706,0.17243560034781694 +fresh-en-1,18,19,V,all_heads,200,0.08250438421964647,0.11047191172838211,0.23382177203893656,0.12838262461125852,200,0.1940002888441086,0.26802419126033783,0.744414210319519,0.3414536836743355 +fresh-en-1,18,19,QKV,weak_half,200,0.03607277311384678,0.04582882113754749,0.06143952682614326,0.04834980919957161,200,0.09681893289089202,0.118901826441288,0.16358237862586975,0.12574409067630768 +fresh-en-1,18,19,QKV,strong_half,200,0.08335503563284874,0.11074938997626305,0.17048740684986113,0.11892115138471127,200,0.21605875790119172,0.27439895272254944,0.4134614080190658,0.2913247724622488 +fresh-en-1,18,19,QKV,random_half,200,0.060306620225310324,0.0834866389632225,0.12245746776461601,0.08840717362239957,200,0.1555040270090103,0.19770444184541702,0.29214662015438075,0.2104254697263241 +fresh-en-1,18,19,QKV,all_heads,200,0.09894444942474366,0.12362789362668991,0.19814366847276688,0.13277632817626,200,0.31665202379226687,0.3933873474597931,0.5794233262538909,0.4170688620209694 +fresh-en-1,18,19,JS,weak_half,200,0.00028739659173879776,0.0006483748438768089,0.0014716432197019453,0.0008046966659094323,200,0.0006307320320047439,0.004201293224468827,0.016128045134246344,0.006247514242859325 +fresh-en-1,18,19,JS,strong_half,200,0.011667023506015539,0.02225372102111578,0.04400902949273585,0.027295302195707336,200,0.004749900149181485,0.014808625914156437,0.028664997965097423,0.016946690250188114 +fresh-en-1,18,19,JS,random_half,200,0.0026307108812034133,0.007177729392424226,0.0225433312356472,0.011059908628521953,200,0.0023203907534480096,0.007849603425711393,0.02304952964186667,0.011019929242902436 +fresh-en-1,18,19,JS,all_heads,200,0.01317417174577713,0.023113549686968327,0.0467867124825716,0.02817574079381302,200,0.011181800533086062,0.02053231094032526,0.03645632937550545,0.023194204496685415 +fresh-en-1,18,19,sqrtJS,weak_half,200,0.016952754557132722,0.025463185273110867,0.03836123198270797,0.026823266111314296,200,0.0669751487672329,0.11005035415291786,0.20602607578039167,0.12288578731939197 +fresh-en-1,18,19,sqrtJS,strong_half,200,0.1080137886106968,0.1491767317056656,0.20977621078491207,0.1585968809016049,200,0.17390958368778228,0.26358993351459503,0.34034043550491333,0.2597278260067105 +fresh-en-1,18,19,sqrtJS,random_half,200,0.05128988921642304,0.08472146838903427,0.15014398992061614,0.09464912596158684,200,0.11044753268361092,0.18393507599830627,0.26525643169879914,0.18850689701735973 +fresh-en-1,18,19,sqrtJS,all_heads,200,0.11477878838777542,0.15203135460615158,0.21630197465419768,0.16153068270534277,200,0.30193769931793213,0.3825230598449707,0.4528791427612305,0.3826136150211096 +fresh-en-1,19,20,Q,weak_half,400,0.020055335760116578,0.045237231999635696,0.09601449593901634,0.052125120621640236,400,0.03522581197321415,0.0855724923312664,0.1677279457449913,0.09480105704627931 +fresh-en-1,19,20,Q,strong_half,400,0.10541421920061111,0.2002575322985649,0.42370031476020825,0.2404438554216176,400,0.12698360681533813,0.2635687589645386,0.567967963218689,0.315416416246444 +fresh-en-1,19,20,Q,random_half,400,0.05309462882578373,0.11167872697114944,0.23741611689329167,0.14009564856998621,400,0.07958442494273187,0.17522764950990677,0.3596063822507859,0.2045334294438362 +fresh-en-1,19,20,Q,all_heads,400,0.12096542865037918,0.22217325866222382,0.4413401335477829,0.2628820433467627,400,0.16694879829883577,0.3607849180698395,0.6931609988212586,0.410217473693192 +fresh-en-1,19,20,K,weak_half,400,0.02373089026659727,0.04101288504898548,0.07946780472993852,0.047759228507056835,400,0.04846536330878735,0.0797160230576992,0.14573340415954592,0.08914559593424201 +fresh-en-1,19,20,K,strong_half,400,0.14686052799224855,0.23109806329011917,0.38704917132854466,0.2539664949849248,400,0.18300481289625167,0.2860434353351593,0.4498937785625458,0.3044107376039028 +fresh-en-1,19,20,K,random_half,400,0.07433910965919495,0.12453599274158478,0.24795479178428653,0.1476036545727402,400,0.10414203405380248,0.1744680181145668,0.3142328202724458,0.19671721584163607 +fresh-en-1,19,20,K,all_heads,400,0.16225437968969345,0.266920804977417,0.44122807085514076,0.2889228591695428,400,0.24076349437236788,0.3753089904785156,0.5617913365364076,0.39355633340775964 +fresh-en-1,19,20,V,weak_half,400,0.02690054401755333,0.047163670882582664,0.08248112052679063,0.05282141955103725,400,0.05348616167902947,0.08201628923416138,0.153989477455616,0.09637866431847214 +fresh-en-1,19,20,V,strong_half,400,0.1764236018061638,0.2465289682149887,0.3216236591339111,0.24791794450953603,400,0.2303718388080597,0.2957165539264679,0.38953552246093753,0.30520592419430614 +fresh-en-1,19,20,V,random_half,400,0.08325937539339066,0.14462488889694214,0.23034160882234575,0.15055040345527232,400,0.11039087101817131,0.19435212016105652,0.2887781918048859,0.20041543115861715 +fresh-en-1,19,20,V,all_heads,400,0.1976504147052765,0.27837684750556946,0.37473527789115935,0.28561449524015187,400,0.2948565185070038,0.3821024000644684,0.5098383665084839,0.40158458802849056 +fresh-en-1,19,20,QKV,weak_half,400,0.030211927741765975,0.049028392881155014,0.0867822051048279,0.05368776506744325,400,0.05643323436379433,0.08845436945557594,0.1578233510255814,0.09824418446049094 +fresh-en-1,19,20,QKV,strong_half,400,0.17469831109046938,0.24496793746948242,0.3541853189468384,0.25925030635669827,400,0.23022768050432205,0.3108224719762802,0.45065078735351566,0.3283278101310134 +fresh-en-1,19,20,QKV,random_half,400,0.08693523481488227,0.1395275890827179,0.2457732126116753,0.15435307613573968,400,0.1207028403878212,0.1972372904419899,0.3349162220954895,0.212870449796319 +fresh-en-1,19,20,QKV,all_heads,400,0.19533516317605973,0.28073807060718536,0.3891092211008072,0.29061810377985237,400,0.28958061039447786,0.41380414366722107,0.5908717215061188,0.4265719954669476 +fresh-en-1,20,21,Q,weak_half,400,0.011196633707731963,0.01916349306702614,0.053231155499815955,0.02557278914260678,400,0.032222453504800797,0.052744315937161446,0.14913512766361237,0.07206435420550406 +fresh-en-1,20,21,Q,strong_half,400,0.026734981313347818,0.047788677737116814,0.15370926260948184,0.06903509186115116,400,0.07141625210642814,0.11843256652355194,0.3471874505281449,0.16391941031441093 +fresh-en-1,20,21,Q,random_half,400,0.01878144443035126,0.03411971963942051,0.09818209782242777,0.0475731286755763,400,0.05156542398035526,0.08562725782394409,0.2402356430888176,0.11610176698304712 +fresh-en-1,20,21,Q,all_heads,400,0.028794750198721887,0.051772767677903175,0.17072072029113772,0.07539859297685325,400,0.10556339174509048,0.17028770595788956,0.5006994247436525,0.23598376426845788 +fresh-en-1,20,21,K,weak_half,400,0.014057555329054594,0.03329697996377945,0.05696576572954656,0.03472992334747687,400,0.04175920076668262,0.09057793393731117,0.13526903688907627,0.09193706802558153 +fresh-en-1,20,21,K,strong_half,400,0.03557929806411266,0.09784767776727676,0.155314938724041,0.09810679198242724,400,0.08455806821584702,0.22414063662290573,0.3389277994632721,0.22260037350468337 +fresh-en-1,20,21,K,random_half,400,0.024819261394441128,0.06814546138048172,0.11199119389057167,0.06968650351744145,400,0.06407569125294686,0.15059003978967667,0.23902006447315216,0.1555986937787384 +fresh-en-1,20,21,K,all_heads,400,0.038586179912090304,0.10798711702227592,0.1626253798604012,0.10652564968448132,400,0.12541414573788642,0.31875115633010864,0.4695759236812592,0.3145374427549541 +fresh-en-1,20,21,V,weak_half,400,0.022645928710699082,0.03378218226134777,0.058599837124347694,0.03833259409060702,400,0.0666954644024372,0.0920448750257492,0.14654957056045534,0.10060668763704599 +fresh-en-1,20,21,V,strong_half,400,0.06070527248084545,0.08942380547523499,0.14785394966602328,0.09719870590604841,400,0.17785485088825226,0.23636244237422943,0.3443977415561676,0.24839538112282752 +fresh-en-1,20,21,V,random_half,400,0.040481336787343024,0.06352852657437325,0.10955837517976766,0.07096631270833313,400,0.11167957410216332,0.1615947186946869,0.25458785593509675,0.17374035205692054 +fresh-en-1,20,21,V,all_heads,400,0.06980053558945656,0.1003243625164032,0.15888779014348983,0.10970477119088173,400,0.25217120945453647,0.3262440264225006,0.48325258493423473,0.34900207035243513 +fresh-en-1,20,21,QKV,weak_half,400,0.02189337871968746,0.033147381618618965,0.05252435989677907,0.03513096733484417,400,0.06106250174343586,0.08704379573464394,0.12910876125097276,0.09396877220831812 +fresh-en-1,20,21,QKV,strong_half,400,0.06082925461232662,0.08788276463747025,0.13574417978525163,0.09430919297039508,400,0.15206177681684493,0.2187458649277687,0.31341068148612977,0.22755627984181045 +fresh-en-1,20,21,QKV,random_half,400,0.0404394369572401,0.06314683333039284,0.1004673004150391,0.06733055203687399,400,0.10034104958176612,0.1524890959262848,0.23358420580625536,0.1592005030438304 +fresh-en-1,20,21,QKV,all_heads,400,0.06775726601481438,0.09717035666108131,0.14664675742387773,0.10407364489510655,400,0.21582522541284563,0.3058822453022003,0.4426770001649857,0.3215250526741147 +fresh-en-1,21,22,Q,weak_half,400,0.009499535709619525,0.02219329308718443,0.03769478611648083,0.0229584945703391,400,0.023415477015078068,0.06156288832426071,0.10394147112965588,0.061775624225847425 +fresh-en-1,21,22,Q,strong_half,400,0.015571229718625547,0.04107335954904556,0.08251423314213753,0.04547044525621459,400,0.041333315521478654,0.11254997923970222,0.20312552005052575,0.11662367063108832 +fresh-en-1,21,22,Q,random_half,400,0.012400286272168161,0.03017550241202116,0.061468683555722256,0.034141808616695926,400,0.032796534523367886,0.08398794755339622,0.1505865216255188,0.08904188805259765 +fresh-en-1,21,22,Q,all_heads,400,0.021092044375836855,0.0503044743090868,0.10006422027945519,0.05483682771911844,400,0.06384461894631387,0.17670702934265137,0.3051427751779557,0.17839929467067123 +fresh-en-1,21,22,K,weak_half,400,0.009791777282953263,0.0292274197563529,0.054760474711656584,0.03134326019557193,400,0.027753277309238912,0.07764839008450508,0.1301201328635216,0.0786181498807855 +fresh-en-1,21,22,K,strong_half,400,0.01767380386590958,0.055155087262392044,0.09332268461585046,0.05718359048943967,400,0.0452721655368805,0.14292284101247787,0.23436719924211505,0.14319355295971037 +fresh-en-1,21,22,K,random_half,400,0.014209394808858633,0.04593614861369133,0.07606789693236352,0.04637154320720583,400,0.03447344116866589,0.11273185908794403,0.18334985375404358,0.11113918682560325 +fresh-en-1,21,22,K,all_heads,400,0.022090245224535468,0.06681781634688377,0.10754911527037622,0.06725669150240719,400,0.07246483713388444,0.22561558336019516,0.3578642725944519,0.22181170331314207 +fresh-en-1,21,22,V,weak_half,400,0.018081592582166194,0.028786756098270416,0.051370843872427956,0.032333475444465874,400,0.04931764453649521,0.06962477415800095,0.12317382842302323,0.07857210380956531 +fresh-en-1,21,22,V,strong_half,400,0.037979501858353616,0.05547008663415909,0.0915533758699894,0.06098222624976188,400,0.09557486698031425,0.1348882094025612,0.23463459163904193,0.14932742958888412 +fresh-en-1,21,22,V,random_half,400,0.02662113439291716,0.042254069820046425,0.08051785677671433,0.04788169359788299,400,0.06984272003173828,0.09988986700773239,0.17954203933477406,0.11410282169468701 +fresh-en-1,21,22,V,all_heads,400,0.04504407308995724,0.06835626065731049,0.10864392220973969,0.07329757874831558,400,0.14645517766475677,0.20409299433231354,0.3581325888633728,0.2278995330631733 +fresh-en-1,21,22,QKV,weak_half,400,0.01600716430693865,0.02751823700964451,0.047140979766845705,0.03028669808758423,400,0.04459880031645298,0.07432401925325394,0.11501509174704554,0.07687315003946424 +fresh-en-1,21,22,QKV,strong_half,400,0.03368146121501923,0.05430906265974045,0.08470789268612862,0.05726587619166821,400,0.08282851725816727,0.14001881331205368,0.21558328419923783,0.1449585392419249 +fresh-en-1,21,22,QKV,random_half,400,0.023362419195473196,0.04290921054780483,0.07060717567801476,0.045074919825419786,400,0.062101644277572636,0.10655131191015244,0.1662536591291428,0.11111930146813392 +fresh-en-1,21,22,QKV,all_heads,400,0.04068434871733189,0.06473659723997116,0.10111721977591515,0.06792558495886623,400,0.1288081720471382,0.21573365479707718,0.32779277563095094,0.22183168843388557 +fresh-en-1,22,23,Q,weak_half,200,0.032375112920999524,0.04311622865498066,0.060389561206102366,0.04472352950833738,200,0.0847306750714779,0.11190426349639893,0.15368748903274534,0.1167450400814414 +fresh-en-1,22,23,Q,strong_half,200,0.10466229692101478,0.13617707043886185,0.1710360750555992,0.13785249155014753,200,0.2591511458158493,0.31125347316265106,0.38652101457118987,0.3190600971877575 +fresh-en-1,22,23,Q,random_half,200,0.06260390281677246,0.0886291041970253,0.1377145320177078,0.09618917444720865,200,0.15593845546245574,0.209866963326931,0.2971095263957977,0.21718349874019624 +fresh-en-1,22,23,Q,all_heads,200,0.11248567700386047,0.1437346488237381,0.17703212052583694,0.1455461272597313,200,0.35923558473587036,0.42900635302066803,0.5183386325836181,0.43580513760447503 +fresh-en-1,22,23,K,weak_half,200,0.023721249774098396,0.034224674105644226,0.049824804067611694,0.03670006334781647,200,0.06345872581005096,0.08442028611898422,0.1199874497950077,0.09132319444790482 +fresh-en-1,22,23,K,strong_half,200,0.10814801082015038,0.12768137454986572,0.21002048552036284,0.1420827630534768,200,0.26403170824050903,0.3104456514120102,0.38652175664901733,0.32220984615385534 +fresh-en-1,22,23,K,random_half,200,0.05321548916399479,0.09376586973667145,0.14730868488550186,0.09693160397931934,200,0.12268906384706497,0.20595058798789978,0.2980729520320892,0.20563207872211933 +fresh-en-1,22,23,K,all_heads,200,0.1114588536322117,0.13515383005142212,0.20705582350492477,0.1464307750761509,200,0.3285584360361099,0.4053556025028229,0.4980300545692443,0.4135330390930176 +fresh-en-1,22,23,V,weak_half,200,0.013882925733923912,0.0182502381503582,0.026541902683675286,0.01935940203256905,200,0.03675885871052742,0.051806241273880005,0.06825781464576722,0.05388862624764443 +fresh-en-1,22,23,V,strong_half,200,0.047567743435502054,0.0811333954334259,0.15947033315896975,0.09335915017873049,200,0.1444399207830429,0.19531748443841934,0.2836032509803772,0.20940759316086768 +fresh-en-1,22,23,V,random_half,200,0.028000675700604917,0.04509470611810684,0.12692201435565947,0.06231444664299488,200,0.07336287125945092,0.11599641293287277,0.20862639248371123,0.1299680894985795 +fresh-en-1,22,23,V,all_heads,200,0.05117474049329758,0.07625066488981247,0.14108106046915037,0.09255334623157978,200,0.1890995293855667,0.25209690630435944,0.3402364253997803,0.26329622030258176 +fresh-en-1,22,23,QKV,weak_half,200,0.025382535345852376,0.033811191096901894,0.047017752751708025,0.0353678544331342,200,0.06550594046711922,0.08896845579147339,0.11797405257821082,0.09240278234705329 +fresh-en-1,22,23,QKV,strong_half,200,0.09828936159610749,0.12323373183608055,0.1795388951897621,0.12933908261358737,200,0.24802139699459075,0.294119194149971,0.36219791769981385,0.3014212240278721 +fresh-en-1,22,23,QKV,random_half,200,0.05216319970786572,0.08294957876205444,0.1284248873591423,0.0891679210588336,200,0.12605671733617782,0.1914069503545761,0.270766618847847,0.19563648011535406 +fresh-en-1,22,23,QKV,all_heads,200,0.10244977623224258,0.12855790555477142,0.17668998092412946,0.13330988194793464,200,0.31656256020069123,0.39031925797462463,0.4747440665960312,0.3938240046799183 +fresh-en-1,22,23,JS,weak_half,200,9.82328238023911e-06,0.0002860648091882467,0.000733790436061099,0.00034281680963431425,200,2.6025114493677396e-05,0.00039887079037725925,0.0012120119179598987,0.0005206021934043292 +fresh-en-1,22,23,JS,strong_half,200,0.00036898800462950023,0.003847413114272058,0.008335836045444012,0.004495912526708707,200,0.0006009698787238449,0.005029736552387476,0.009270998556166886,0.00520713249919936 +fresh-en-1,22,23,JS,random_half,200,0.00022958591289352627,0.003315136069431901,0.00823884205892682,0.003923301479262591,200,0.00024643279757583517,0.0022952641593292356,0.0054657832253724335,0.0028045701339215155 +fresh-en-1,22,23,JS,all_heads,200,0.000480932928621769,0.00449235481210053,0.009812514856457706,0.005033490254118078,200,0.0006548373959958554,0.005710975034162402,0.010046254750341177,0.0057277347047784135 +fresh-en-1,22,23,sqrtJS,weak_half,200,0.003133736154995859,0.016913301311433315,0.027088096179068086,0.016482633443083615,200,0.012570316344499588,0.04907296597957611,0.07236151844263077,0.046293843681924046 +fresh-en-1,22,23,sqrtJS,strong_half,200,0.019208510220050812,0.06202738359570503,0.09130079969763756,0.06087514445884153,200,0.051990200579166417,0.16245626658201218,0.21092731058597564,0.15035413179546595 +fresh-en-1,22,23,sqrtJS,random_half,200,0.0151518183760345,0.05757705867290497,0.09076790660619735,0.056238489607349035,200,0.03276577070355415,0.10424651205539703,0.15573255270719527,0.09874753409996628 +fresh-en-1,22,23,sqrtJS,all_heads,200,0.021929077431559564,0.06702489033341408,0.09905066192150114,0.06478990578791127,200,0.06319581791758537,0.21820172667503357,0.2689065098762512,0.1966479748673737 +fresh-en-2,0,1,Q,weak_half,400,0.18981192559003832,0.30438530445098877,0.4745282769203186,0.3214992294833064,400,0.4253257870674133,0.7224414348602295,1.0455863237380985,0.7399280648678541 +fresh-en-2,0,1,Q,strong_half,400,0.7713914334774018,1.119035542011261,1.47206209897995,1.1273243824392558,400,1.9413662791252138,2.999807119369507,3.957393074035645,2.9461909037828447 +fresh-en-2,0,1,Q,random_half,400,0.5585763990879059,1.0459519028663635,1.4320171356201172,1.0058840183913709,400,1.0437860488891602,1.8176250457763672,2.7251336812973035,1.8400868697464465 +fresh-en-2,0,1,Q,all_heads,400,1.0137540221214294,1.3023966550827026,1.6032708048820499,1.307453941255808,400,2.4726752042770386,3.7375751733779907,4.850858545303344,3.6861189687252045 +fresh-en-2,0,1,K,weak_half,400,0.23804241120815278,0.3619637191295624,0.5365876734256745,0.38221794471144677,400,0.5819603621959687,0.8125706315040588,1.1411016941070558,0.8406451546400785 +fresh-en-2,0,1,K,strong_half,400,0.8405694603919983,1.0646045207977295,1.3773112535476686,1.085583837777376,400,2.2374348640441895,2.7914509773254395,3.681853413581848,2.9039069122076033 +fresh-en-2,0,1,K,random_half,400,0.6691918671131134,0.9862692952156067,1.347035324573517,0.9958072275668383,400,1.22822345495224,1.7977826595306396,2.5937507390975956,1.8660968127846718 +fresh-en-2,0,1,K,all_heads,400,1.060123574733734,1.2868735194206238,1.5202052831649782,1.284539031535387,400,2.9589993953704834,3.614795207977295,4.718218660354614,3.744552076458931 +fresh-en-2,0,1,V,weak_half,400,0.10627884045243265,0.2506295442581177,0.4667730152606965,0.2828158756624907,400,0.2977455317974091,0.5678088963031769,0.9337635636329656,0.6037796954810619 +fresh-en-2,0,1,V,strong_half,400,0.5490926802158356,0.8540489077568054,1.3967276453971866,0.9229900197684765,400,1.2621386766433715,2.090788960456848,3.529253005981447,2.3325862143933773 +fresh-en-2,0,1,V,random_half,400,0.3504885405302048,0.7358205616474152,1.450324130058289,0.8768366070464253,400,0.6998965978622437,1.3255137205123901,2.321849346160889,1.465379197075963 +fresh-en-2,0,1,V,all_heads,400,0.6910390794277191,1.0958972573280334,1.5811966300010682,1.1100770066678525,400,1.601429045200348,2.7185384035110474,4.425310707092285,2.9363659144937992 +fresh-en-2,0,1,QKV,weak_half,400,0.23150804787874224,0.3343365788459778,0.4628632068634034,0.3453738921135664,400,0.5610997676849365,0.7536446750164032,1.0103684902191163,0.7697968394309282 +fresh-en-2,0,1,QKV,strong_half,400,0.8393572807312012,1.0437911748886108,1.3437451124191284,1.0700127211213113,400,2.223258304595947,2.7742549180984497,3.6279370069503787,2.893503189086914 +fresh-en-2,0,1,QKV,random_half,400,0.6662041664123536,0.9685291647911072,1.328384339809418,1.0039024426043033,400,1.229683303833008,1.7470809817314148,2.551129770278931,1.828443867266178 +fresh-en-2,0,1,QKV,all_heads,400,1.0390419006347655,1.241932988166809,1.4820349931716919,1.2529276865720749,400,2.877012515068054,3.5474698543548584,4.493176889419556,3.66330001950264 +fresh-en-2,1,2,Q,weak_half,400,0.04966536723077298,0.09323367103934288,0.1771605074405671,0.10539483960717916,400,0.1231757014989853,0.25049515068531036,0.4286495625972748,0.2643243132252246 +fresh-en-2,1,2,Q,strong_half,400,0.20340016931295396,0.3998543322086334,0.6681329965591432,0.43458795983344317,400,0.5054378867149353,0.9864613711833954,1.4355821251869203,0.9739338751882315 +fresh-en-2,1,2,Q,random_half,400,0.13441669493913652,0.25483568012714386,0.45536783933639524,0.2789147196151316,400,0.310806131362915,0.5914080739021301,0.9350887298583984,0.6152953441813588 +fresh-en-2,1,2,Q,all_heads,400,0.21630829721689224,0.4318287521600723,0.7470322668552399,0.46511321417987345,400,0.6467387557029725,1.2311389446258545,1.842737364768982,1.2382581877708434 +fresh-en-2,1,2,K,weak_half,400,0.06282230094075203,0.11339181661605835,0.19854123294353487,0.12631659057922662,400,0.14455468207597733,0.28196004033088684,0.45711011290550246,0.29457505606114864 +fresh-en-2,1,2,K,strong_half,400,0.2781426548957825,0.41335056722164154,0.647815752029419,0.441851559728384,400,0.6791548013687134,0.9697889089584351,1.4401088714599615,1.0233507636934518 +fresh-en-2,1,2,K,random_half,400,0.17642932683229448,0.2847895175218582,0.45629326999187475,0.30393513549119233,400,0.39638811647891997,0.6128599345684052,0.9726890683174133,0.6531811596080661 +fresh-en-2,1,2,K,all_heads,400,0.3006390631198883,0.45238251984119415,0.6940921008586886,0.4825432312116027,400,0.8913047969341278,1.2562485337257385,1.8419503450393682,1.3179258199781179 +fresh-en-2,1,2,V,weak_half,400,0.039820257574319846,0.10014444589614868,0.2252603858709336,0.12056123815011233,400,0.09140035808086396,0.22909922152757645,0.48582843542099013,0.2656730801658705 +fresh-en-2,1,2,V,strong_half,400,0.17217100858688356,0.35721972584724426,0.6971244812011722,0.4117434100061655,400,0.4317542612552643,0.8542904257774353,1.5939001321792603,0.9694790058583022 +fresh-en-2,1,2,V,random_half,400,0.1161925919353962,0.24929118156433105,0.48809911906719206,0.28397300293669103,400,0.26720134913921356,0.5368628799915314,1.0559662938117986,0.6115258574485779 +fresh-en-2,1,2,V,all_heads,400,0.19875250905752181,0.4004882127046585,0.7866175413131715,0.4545604870840907,400,0.540332567691803,1.1034665703773499,2.0374572277069096,1.2351520878076554 +fresh-en-2,1,2,QKV,weak_half,400,0.06346190944314004,0.11281995102763176,0.1944100871682167,0.12411719001829624,400,0.14875513017177583,0.280353844165802,0.4487576723098755,0.2924985410645604 +fresh-en-2,1,2,QKV,strong_half,400,0.27848084568977355,0.41599999368190765,0.6457281351089478,0.44724048029631375,400,0.6951691508293152,1.0106902122497559,1.446615755558014,1.0536176946759224 +fresh-en-2,1,2,QKV,random_half,400,0.17742505073547363,0.2821838706731796,0.45840083956718447,0.3020446793735027,400,0.40486094653606414,0.6172370910644531,0.9952882051467896,0.6666365534812212 +fresh-en-2,1,2,QKV,all_heads,400,0.3073578953742981,0.45682068169116974,0.7043636739253999,0.48631976276636124,400,0.878587543964386,1.2940925359725952,1.877209830284119,1.3461162407696248 +fresh-en-2,2,3,Q,weak_half,200,0.07289785072207451,0.13372255861759186,0.19169748723506927,0.13354009222239255,200,0.166483348608017,0.3209023177623749,0.46703161001205445,0.3157518841326237 +fresh-en-2,2,3,Q,strong_half,200,0.4595774203538895,0.5765953063964844,0.7288557171821595,0.588307632803917,200,0.8266426920890808,1.1437352895736694,1.4461750745773316,1.1563536486029624 +fresh-en-2,2,3,Q,random_half,200,0.1922074243426323,0.39628133177757263,0.5601939916610718,0.38304432954639195,200,0.4108307331800461,0.7704713642597198,1.062635362148285,0.7483570852130651 +fresh-en-2,2,3,Q,all_heads,200,0.5212890982627869,0.6439124643802643,0.7730342447757721,0.6488065369427204,200,1.0245461106300355,1.4682307839393616,1.8709295749664305,1.472105537354946 +fresh-en-2,2,3,K,weak_half,200,0.07933490425348286,0.13581663370132446,0.16923540830612183,0.1283938106894493,200,0.1798241615295411,0.30905960500240326,0.41204237937927246,0.2995810043811798 +fresh-en-2,2,3,K,strong_half,200,0.41720238626003264,0.4926001727581024,0.5945168852806091,0.49618090912699697,200,0.8891628444194795,1.0765504837036133,1.3154743909835815,1.0875110778212547 +fresh-en-2,2,3,K,random_half,200,0.2079329714179039,0.356093093752861,0.4504793643951416,0.34390586920082566,200,0.4500078111886978,0.7218021154403687,0.9197448372840881,0.7032462854683399 +fresh-en-2,2,3,K,all_heads,200,0.46375351250171665,0.557563990354538,0.6474413335323334,0.5541302925348282,200,1.0471121788024904,1.3830288648605347,1.682318818569183,1.387092077732086 +fresh-en-2,2,3,V,weak_half,200,0.10586665794253351,0.16299565136432648,0.23649485707283008,0.1671868707612157,200,0.22541494369506837,0.39080843329429626,0.5740876734256745,0.38496061250567437 +fresh-en-2,2,3,V,strong_half,200,0.4674473732709885,0.552510678768158,0.7557965099811554,0.5867945177853108,200,1.0261149466037751,1.2545182704925537,1.8044749975204457,1.3466503995656967 +fresh-en-2,2,3,V,random_half,200,0.2476434588432312,0.41737981140613556,0.5956592619419098,0.42025579571723937,200,0.5466697812080383,0.8343739211559296,1.252881121635437,0.8743665853142738 +fresh-en-2,2,3,V,all_heads,200,0.5205116450786591,0.6224192380905151,0.8316396176815032,0.6523989862203599,200,1.3195908784866333,1.604778230190277,2.436248898506163,1.7316110157966613 +fresh-en-2,2,3,QKV,weak_half,200,0.08705131858587266,0.14840421080589294,0.1996922343969345,0.14519645947963,200,0.1904606342315674,0.3499113470315933,0.4864329397678375,0.3400423405319452 +fresh-en-2,2,3,QKV,strong_half,200,0.4667528152465821,0.5479975342750549,0.6827597737312316,0.5618404626846314,200,0.9528383255004883,1.1747262477874756,1.5242294549942017,1.2157843512296678 +fresh-en-2,2,3,QKV,random_half,200,0.22591662555932998,0.3989194929599762,0.5204320073127746,0.3867190781235695,200,0.49278026521205903,0.7934269905090332,1.060936653614044,0.7881524117290973 +fresh-en-2,2,3,QKV,all_heads,200,0.5183119058609009,0.6166341304779053,0.7411625862121582,0.6231636847555637,200,1.1746613740921021,1.5100290179252625,2.0858570098876954,1.5558266961574554 +fresh-en-2,2,3,JS,weak_half,200,0.0017874031676910818,0.005385936237871647,0.012731500528752803,0.006552178232668666,200,0.0031506994273513556,0.006335825193673372,0.013189065363258123,0.007201125471037812 +fresh-en-2,2,3,JS,strong_half,200,0.04199272319674492,0.08988497778773308,0.16329088658094407,0.09681790921837091,200,0.03897917829453945,0.06828472763299942,0.14045877158641815,0.07974548957310618 +fresh-en-2,2,3,JS,random_half,200,0.027563737146556376,0.05898197367787361,0.1289644330739975,0.0689963699132204,200,0.009941180050373078,0.0377902053296566,0.08586280420422551,0.04448760825325735 +fresh-en-2,2,3,JS,all_heads,200,0.04705630503594876,0.10175274685025215,0.19551064521074293,0.11343342888634651,200,0.04602197483181954,0.07628454267978668,0.14738180488348007,0.08694661519955843 +fresh-en-2,2,3,sqrtJS,weak_half,200,0.04227702990174294,0.07338893786072731,0.1128337636590004,0.07590119233354926,200,0.14482425451278685,0.1988639459013939,0.2817451566457748,0.20609549369663 +fresh-en-2,2,3,sqrtJS,strong_half,200,0.2049207091331482,0.299807533621788,0.40409243404865264,0.3013616531714797,200,0.44698799252510074,0.6061300337314606,0.8016758501529694,0.6104789521545172 +fresh-en-2,2,3,sqrtJS,random_half,200,0.1660232812166214,0.2428617998957634,0.3591160774230957,0.25151635374873876,200,0.23426026999950408,0.4062005281448364,0.5750127971172333,0.41082159388810396 +fresh-en-2,2,3,sqrtJS,all_heads,200,0.21692346632480622,0.3189856708049774,0.44216451346874236,0.3264924431592226,200,0.6190151572227478,0.8163227438926697,1.0459965586662292,0.8165744490921497 +fresh-en-2,3,4,Q,weak_half,400,0.1165211595594883,0.20810014754533768,0.3397079169750214,0.21907647563144564,400,0.2335159420967102,0.41976524889469147,0.6384933650493624,0.4313083514943719 +fresh-en-2,3,4,Q,strong_half,400,0.14438361823558807,0.2867623418569565,0.6526487588882449,0.3525429511629045,400,0.40879604816436765,0.7814792692661285,2.1867457389831544,0.9881023425608874 +fresh-en-2,3,4,Q,random_half,400,0.140219184756279,0.2603971064090729,0.6576477348804479,0.32857244653627277,400,0.30907062292099,0.593238353729248,1.2897993326187136,0.7124491903185844 +fresh-en-2,3,4,Q,all_heads,400,0.16099763214588164,0.31922832131385803,0.6942291498184209,0.3799943268112838,400,0.6649561524391174,1.2199535369873047,2.615574288368226,1.4194106931239368 +fresh-en-2,3,4,K,weak_half,400,0.13586917668581008,0.20416072010993958,0.32327749133110056,0.22174933508038522,400,0.2880915731191635,0.38677816092967987,0.6015399336814883,0.4230397238582373 +fresh-en-2,3,4,K,strong_half,400,0.16059805154800416,0.24850211292505264,0.4161278188228608,0.27077258192002773,400,0.4605114161968231,0.6804436147212982,1.1599654316902162,0.7562835172563791 +fresh-en-2,3,4,K,random_half,400,0.1571640431880951,0.24308013916015625,0.39890367686748507,0.27134674103930595,400,0.37225216925144194,0.5257269740104675,0.9204198360443115,0.5936300794780255 +fresh-en-2,3,4,K,all_heads,400,0.1846910759806633,0.2721366435289383,0.4252965569496156,0.2931641483306885,400,0.7907507240772248,1.0601140260696411,1.7786273241043091,1.1793232437968255 +fresh-en-2,3,4,V,weak_half,400,0.09503980427980423,0.18100082129240036,0.38082656860351577,0.21498120124451817,400,0.1973787233233452,0.3671356588602066,0.6592840552330017,0.40766293300315737 +fresh-en-2,3,4,V,strong_half,400,0.10356362462043762,0.2324906438589096,0.46204690635204315,0.2650806562136859,400,0.4328334212303162,0.7784209251403809,1.3467008948326111,0.854417009204626 +fresh-en-2,3,4,V,random_half,400,0.10809355452656746,0.24158594012260437,0.44507176876068116,0.2647682765685022,400,0.3139983713626862,0.5498670637607574,1.078375256061554,0.625490050278604 +fresh-en-2,3,4,V,all_heads,400,0.11934929937124253,0.24848222732543945,0.4520741015672684,0.2761754441075027,400,0.6409734070301056,1.1392776370048523,2.007600212097168,1.2620799472182989 +fresh-en-2,3,4,QKV,weak_half,400,0.1357751876115799,0.2020711973309517,0.34501618444919596,0.2254261015728116,400,0.28636795878410337,0.4015830457210541,0.6285447120666504,0.4354360576719046 +fresh-en-2,3,4,QKV,strong_half,400,0.1660889208316803,0.2852608859539032,0.498749190568924,0.31520057694986464,400,0.5239201903343201,0.8100369870662689,1.5206071376800538,0.9239703406393528 +fresh-en-2,3,4,QKV,random_half,400,0.16678081899881364,0.2732464522123337,0.4968442410230638,0.3053321359679103,400,0.38354486525058745,0.6086342930793762,1.1045093894004825,0.6815331147611141 +fresh-en-2,3,4,QKV,all_heads,400,0.1945397585630417,0.2991437166929245,0.5322174787521363,0.3355102537944913,400,0.8190693378448487,1.2374368906021118,2.0703954696655273,1.359406392723322 +fresh-en-2,4,5,Q,weak_half,400,0.06481311097741127,0.1070447526872158,0.1592010542750359,0.10974143390543759,400,0.19021759927272797,0.2897418141365051,0.4306955605745316,0.29934812534600497 +fresh-en-2,4,5,Q,strong_half,400,0.19305409044027327,0.3026126027107239,0.470282855629921,0.31854032307863234,400,0.5092226922512054,0.8032291233539581,1.155049550533295,0.8259983222186565 +fresh-en-2,4,5,Q,random_half,400,0.11941771730780602,0.21756771206855774,0.36713487505912784,0.23113286730833352,400,0.31053170263767244,0.5381883382797241,0.8248584866523744,0.5581494920141995 +fresh-en-2,4,5,Q,all_heads,400,0.2099371924996376,0.3206407278776169,0.4984588742256165,0.34203521616756916,400,0.725872004032135,1.0927817821502686,1.5961486816406252,1.1253464508801698 +fresh-en-2,4,5,K,weak_half,400,0.07062573879957199,0.11467186361551285,0.1669246792793274,0.11572039125487209,400,0.1823841154575348,0.2952483743429184,0.4140547037124634,0.2973948811925948 +fresh-en-2,4,5,K,strong_half,400,0.17099130600690843,0.27153180539608,0.3913269013166428,0.2794873511698097,400,0.5016963720321657,0.7333408892154694,1.0470094323158265,0.7530355694890022 +fresh-en-2,4,5,K,random_half,400,0.11846136599779129,0.20244452357292175,0.3094224363565445,0.21050178760662674,400,0.30455918908119206,0.5083464086055756,0.7388451993465424,0.519619707595557 +fresh-en-2,4,5,K,all_heads,400,0.1888374075293541,0.29797162115573883,0.4335693955421448,0.30180296121165157,400,0.7027743399143219,1.027060627937317,1.4451884388923646,1.0504304495453836 +fresh-en-2,4,5,V,weak_half,400,0.035824692249298094,0.07657012715935707,0.158488067984581,0.09004117885604501,400,0.1077748477458954,0.1996694877743721,0.3634045362472535,0.22340031635016203 +fresh-en-2,4,5,V,strong_half,400,0.10232129469513894,0.20165687799453735,0.40529019832611096,0.24251914280466735,400,0.2747326880693436,0.5485850274562836,1.0593015432357789,0.6261368382722139 +fresh-en-2,4,5,V,random_half,400,0.06539347134530546,0.14173901826143265,0.3094674348831178,0.1727631646906957,400,0.1917420089244843,0.3619445264339447,0.7297376453876496,0.4228189871646464 +fresh-en-2,4,5,V,all_heads,400,0.11705059036612511,0.23315579444169998,0.4456813216209412,0.2674242053180933,400,0.39125583171844486,0.7567286193370819,1.4041822195053102,0.8495371546596289 +fresh-en-2,4,5,QKV,weak_half,400,0.07261811196804047,0.105468038469553,0.15749695897102356,0.10994985333643853,400,0.19910903573036193,0.2803618907928467,0.3958923816680908,0.29060350853949785 +fresh-en-2,4,5,QKV,strong_half,400,0.19977831542491914,0.2724078744649887,0.4037604719400406,0.29316114688292144,400,0.5373436808586121,0.7418304681777954,1.0721104502677918,0.7861468168348074 +fresh-en-2,4,5,QKV,random_half,400,0.13700721710920333,0.2029600366950035,0.3150731801986695,0.21579476147890092,400,0.33880961537361143,0.5083339810371399,0.7499371469020846,0.5340990379825235 +fresh-en-2,4,5,QKV,all_heads,400,0.21677678376436235,0.2979087084531784,0.4289210826158524,0.31668629698455336,400,0.7460156798362733,1.0318759679794312,1.4543714642524719,1.076750323176384 +fresh-en-2,5,6,Q,weak_half,400,0.08264838084578514,0.13351446390151978,0.18887250870466232,0.1360455384850502,400,0.2325816869735718,0.3688170313835144,0.4882249981164933,0.36739657185971736 +fresh-en-2,5,6,Q,strong_half,400,0.18772378563880923,0.29587236046791077,0.4334669798612595,0.3028398212790489,400,0.5960947573184967,0.8648049831390381,1.1001763701438905,0.8491541681438685 +fresh-en-2,5,6,Q,random_half,400,0.1399589478969574,0.21413318067789078,0.33956786692142493,0.2317296143155545,400,0.3974854201078415,0.5869112014770508,0.8699968457221985,0.6130903580412269 +fresh-en-2,5,6,Q,all_heads,400,0.203337599337101,0.31992103159427643,0.4883055269718171,0.3370681281387806,400,0.8515744149684906,1.232835829257965,1.57544664144516,1.2165507408976555 +fresh-en-2,5,6,K,weak_half,400,0.1070883683860302,0.14540735632181168,0.20651666074991226,0.1525458437204361,400,0.27972509264945983,0.3864535838365555,0.524495804309845,0.3969594720378518 +fresh-en-2,5,6,K,strong_half,400,0.21440026462078096,0.29674549400806427,0.4000360816717148,0.3037652421370149,400,0.6222273707389832,0.8070858716964722,1.0938635230064393,0.8353821855783462 +fresh-en-2,5,6,K,random_half,400,0.16834143102169039,0.24032830446958542,0.34249286353588104,0.248766655176878,400,0.4344429045915604,0.5963755249977112,0.8393229722976686,0.6175591893494129 +fresh-en-2,5,6,K,all_heads,400,0.2473738953471184,0.3208625465631485,0.45002968013286593,0.33905934225767853,400,0.9205420076847076,1.1929860711097717,1.6117078304290773,1.232341657280922 +fresh-en-2,5,6,V,weak_half,400,0.05995054505765438,0.12322742119431496,0.239916068315506,0.1397804959118366,400,0.16258048564195632,0.3202803134918213,0.503335189819336,0.33453178154304625 +fresh-en-2,5,6,V,strong_half,400,0.1424448385834694,0.26767173409461975,0.4591150760650635,0.2861292095854878,400,0.41034115552902223,0.7252813577651978,1.1671292781829836,0.7647753600031137 +fresh-en-2,5,6,V,random_half,400,0.09548224061727524,0.20278050750494003,0.4027047097682953,0.22365778291597962,400,0.27594787776470187,0.5108919143676758,0.8896234750747681,0.5531269352510572 +fresh-en-2,5,6,V,all_heads,400,0.1625249445438385,0.3138740062713623,0.5627337396144868,0.3415474540926516,400,0.5877458155155182,1.041554868221283,1.6692363619804385,1.0993071387708186 +fresh-en-2,5,6,QKV,weak_half,400,0.10849354118108749,0.1432236135005951,0.19212197661399844,0.14830812823027373,400,0.28370408713817596,0.3869010955095291,0.478366357088089,0.38529872719198466 +fresh-en-2,5,6,QKV,strong_half,400,0.22818659842014313,0.3005172610282898,0.40092963278293614,0.3074360596016049,400,0.6707326650619507,0.8585898876190186,1.059073543548584,0.8637860921025277 +fresh-en-2,5,6,QKV,random_half,400,0.1670565828680992,0.2359314188361168,0.33458193838596345,0.24471782971173525,400,0.45400914549827576,0.6210423707962036,0.8158775091171264,0.6283381981402636 +fresh-en-2,5,6,QKV,all_heads,400,0.25516014695167544,0.33772411942481995,0.4692617565393448,0.35088143792003396,400,0.9811203181743622,1.2410852909088135,1.5243494153022767,1.2490848207473755 +fresh-en-2,6,7,Q,weak_half,200,0.0541684988886118,0.08230233564972878,0.12165215387940406,0.08573357237502932,200,0.14045883119106292,0.2097441405057907,0.312593200802803,0.2177259984612465 +fresh-en-2,6,7,Q,strong_half,200,0.2541630804538727,0.3081369400024414,0.38694228827953336,0.31333972588181497,200,0.5701021790504456,0.6875967979431152,0.9049644231796264,0.71695769444108 +fresh-en-2,6,7,Q,random_half,200,0.11937392875552177,0.2050500512123108,0.3041515588760376,0.20788828011602165,200,0.27806739807128905,0.4421647638082504,0.6577842533588409,0.45736666344106197 +fresh-en-2,6,7,Q,all_heads,200,0.2692623555660248,0.32370932400226593,0.4142548471689224,0.33619142018258574,200,0.7084877252578735,0.8912785947322845,1.1909089565277098,0.9346836894750595 +fresh-en-2,6,7,K,weak_half,200,0.06079959608614445,0.0905604176223278,0.12237883210182189,0.09155363455414772,200,0.15486166030168533,0.22660616040229797,0.3119183242321014,0.2302660423517227 +fresh-en-2,6,7,K,strong_half,200,0.23013105988502502,0.29043175280094147,0.3355171710252761,0.287416575551033,200,0.5989623188972474,0.7255833446979523,0.8896279394626616,0.729013899564743 +fresh-en-2,6,7,K,random_half,200,0.1321961596608162,0.20833222568035126,0.2695443361997604,0.20494284406304358,200,0.3203566551208496,0.4583234190940857,0.6163037419319152,0.47057265862822534 +fresh-en-2,6,7,K,all_heads,200,0.2494346410036087,0.31031931936740875,0.3595847845077514,0.3110654228925705,200,0.7711761653423309,0.9527064561843872,1.1831207036972045,0.9592799365520477 +fresh-en-2,6,7,V,weak_half,200,0.08185150474309921,0.11674383282661438,0.16697180271148682,0.12082224892452359,200,0.2194169819355011,0.29881560802459717,0.41142529249191284,0.3104029117524624 +fresh-en-2,6,7,V,strong_half,200,0.3195340991020203,0.3749060779809952,0.4297667145729065,0.37851624846458437,200,0.7431013107299805,0.8971426486968994,1.1175977826118468,0.9300652986764908 +fresh-en-2,6,7,V,random_half,200,0.16308525651693345,0.26917050778865814,0.3495364785194397,0.26250014185905457,200,0.4092594265937805,0.6033527255058289,0.8141948580741882,0.6065441398322582 +fresh-en-2,6,7,V,all_heads,200,0.33838528096675874,0.40397462248802185,0.476686218380928,0.4077101922035217,200,0.9653935492038727,1.2215299606323242,1.4691833138465875,1.2404682195186616 +fresh-en-2,6,7,QKV,weak_half,200,0.06829226091504097,0.09913702681660652,0.13199470192193985,0.10129719074815512,200,0.17909177392721176,0.2552284896373749,0.34154532551765443,0.25892243832349776 +fresh-en-2,6,7,QKV,strong_half,200,0.28316350281238556,0.32674264907836914,0.38016044497489926,0.33008201360702516,200,0.6572286486625671,0.8080477118492126,0.9595184445381164,0.8069310000538826 +fresh-en-2,6,7,QKV,random_half,200,0.1463568612933159,0.2318064570426941,0.304271525144577,0.2281413869559765,200,0.3474537938833237,0.5183075368404388,0.6871095538139342,0.5217808823287488 +fresh-en-2,6,7,QKV,all_heads,200,0.2958098441362381,0.3536299169063568,0.4158825308084488,0.3555362968146801,200,0.843960440158844,1.0644298195838928,1.2615623474121094,1.0658534321188926 +fresh-en-2,6,7,JS,weak_half,200,0.0007366604637354614,0.0024305033730342984,0.007371620927006006,0.003282853820710443,200,0.0013481900095939636,0.004457286559045315,0.01036885529756546,0.005186919962143293 +fresh-en-2,6,7,JS,strong_half,200,0.0223995978012681,0.051565395668148994,0.09334684163331985,0.05509985254146159,200,0.020869515649974347,0.034577758982777596,0.0661432385444641,0.03915219245012849 +fresh-en-2,6,7,JS,random_half,200,0.009690203703939916,0.024823913350701332,0.05326978750526905,0.02766336036089342,200,0.007052322430536151,0.01838310994207859,0.03529587946832179,0.020941900154866744 +fresh-en-2,6,7,JS,all_heads,200,0.025110340863466262,0.062209269031882286,0.10665183216333389,0.06570190838072448,200,0.023922488652169706,0.04078545793890953,0.07289505451917648,0.04433911259751767 +fresh-en-2,6,7,sqrtJS,weak_half,200,0.027140399254858494,0.04929918050765991,0.08585812747478486,0.05286826235242188,200,0.09014794826507569,0.16189640015363693,0.24765936583280562,0.16456015571951865 +fresh-en-2,6,7,sqrtJS,strong_half,200,0.1496642917394638,0.22706738859415054,0.3055270344018936,0.22666387364268303,200,0.33806535601615906,0.4619887173175812,0.6223118603229523,0.4723343429714441 +fresh-en-2,6,7,sqrtJS,random_half,200,0.09843652769923211,0.1575555056333542,0.23080243915319443,0.15872665962204338,200,0.19253715872764587,0.3086535632610321,0.43117988407611846,0.3107671199925244 +fresh-en-2,6,7,sqrtJS,all_heads,200,0.158462318778038,0.24941784143447876,0.32657572627067566,0.24774312000721693,200,0.4461918562650681,0.6388791799545288,0.8278976023197174,0.6368944941461087 +fresh-en-2,7,8,Q,weak_half,400,0.13110366612672805,0.21094738692045212,0.39187355339527136,0.23819161828607321,400,0.2852696359157562,0.4717352092266083,0.9065812170505525,0.5382106870785356 +fresh-en-2,7,8,Q,strong_half,400,0.1765700474381447,0.2616458088159561,0.5341376841068268,0.33208504946902395,400,0.6309951066970825,1.0196323990821838,1.9781412839889527,1.1696488801389933 +fresh-en-2,7,8,Q,random_half,400,0.18055093437433245,0.29455070197582245,0.546913880109787,0.339389652851969,400,0.44131107330322267,0.7469545602798462,1.5200919151306151,0.849703226685524 +fresh-en-2,7,8,Q,all_heads,400,0.19345320165157318,0.2768062353134155,0.44761978089809423,0.3122830041497946,400,0.9475403606891633,1.5199984908103943,2.8486045360565186,1.7078595685958862 +fresh-en-2,7,8,K,weak_half,400,0.05971905663609506,0.16648772358894348,0.318999257683754,0.18326670535840095,400,0.10525479614734651,0.3962721824645996,0.707092297077179,0.4226870820717886 +fresh-en-2,7,8,K,strong_half,400,0.06083818450570107,0.2289288565516472,0.3686465173959732,0.24848003847524525,400,0.17734988629817963,0.8486134111881256,1.3743295669555669,0.870980151630938 +fresh-en-2,7,8,K,random_half,400,0.09407174214720727,0.28564855456352234,0.48081415593624116,0.3016776374913752,400,0.1405219152569771,0.607193112373352,1.0680564165115356,0.6518659908231348 +fresh-en-2,7,8,K,all_heads,400,0.06735809594392776,0.22940748929977417,0.3576667726039887,0.23814673310611398,400,0.2791970998048782,1.2571341395378113,2.025203394889832,1.2936672349087894 +fresh-en-2,7,8,V,weak_half,400,0.08698565065860749,0.15835604816675186,0.3136553347110748,0.1828484263084829,400,0.1629799157381058,0.30963248014450073,0.6034546971321106,0.35466510577127336 +fresh-en-2,7,8,V,strong_half,400,0.09565408453345299,0.19631103426218033,0.3952824711799622,0.23153432722203435,400,0.3670838475227356,0.8591986894607544,1.5265558362007141,0.9187744932249189 +fresh-en-2,7,8,V,random_half,400,0.1144007109105587,0.23974020034074783,0.5360902726650238,0.2990356118325144,400,0.2797063767910004,0.5692319571971893,1.0924185395240784,0.6371907974407077 +fresh-en-2,7,8,V,all_heads,400,0.10506940558552742,0.19328629970550537,0.3294906467199326,0.2085337904561311,400,0.5560215830802918,1.20391446352005,2.0314424514770506,1.273439597338438 +fresh-en-2,7,8,QKV,weak_half,400,0.12474201247096063,0.18998130410909653,0.33136106431484225,0.21159961046651005,400,0.2664622098207474,0.4196581393480301,0.7522706210613251,0.46725595716387036 +fresh-en-2,7,8,QKV,strong_half,400,0.16705073714256286,0.24743526428937912,0.4395806580781937,0.28951617574319244,400,0.6065186381340028,1.0038425922393799,1.5280829787254335,1.054482866153121 +fresh-en-2,7,8,QKV,random_half,400,0.18858530074357988,0.302360400557518,0.5164019703865051,0.3345734969154,400,0.41624838411808013,0.6799852550029755,1.190171003341675,0.760482302904129 +fresh-en-2,7,8,QKV,all_heads,400,0.17358650267124176,0.25597040355205536,0.34383021593093877,0.2669654383137822,400,0.8897886574268341,1.438947319984436,2.23228235244751,1.5217388217151164 +fresh-en-2,8,9,Q,weak_half,400,0.07489821314811707,0.11516931280493736,0.15813674926757815,0.11678670703433454,400,0.21274705976247787,0.3236849308013916,0.4388548940420151,0.32572737272828817 +fresh-en-2,8,9,Q,strong_half,400,0.15971188098192216,0.23987457901239395,0.37001143693923955,0.25776736814528706,400,0.46704228520393376,0.6990979611873627,0.9878741800785065,0.7185790818184614 +fresh-en-2,8,9,Q,random_half,400,0.12105938345193863,0.18818673491477966,0.2835905253887177,0.1984751001559198,400,0.321076774597168,0.5116079449653625,0.7058013737201692,0.5211422502994537 +fresh-en-2,8,9,Q,all_heads,400,0.18068428933620453,0.26805251836776733,0.40221258997917175,0.28393224367871883,400,0.71153604388237,1.0439937710762024,1.3857736349105836,1.0443064542114735 +fresh-en-2,8,9,K,weak_half,400,0.07168342471122742,0.11726129800081253,0.1813877195119858,0.12278681077063083,400,0.19563512653112414,0.3087916821241379,0.4319230437278748,0.31523568999022245 +fresh-en-2,8,9,K,strong_half,400,0.14508408457040786,0.22007279843091965,0.3132781714200974,0.2269410969130695,400,0.4656625419855118,0.6566390991210938,0.887118399143219,0.6646402641385794 +fresh-en-2,8,9,K,random_half,400,0.11550938561558724,0.18396416306495667,0.2653342604637146,0.18966359732672572,400,0.31869955360889435,0.47293585538864136,0.6763127326965332,0.48706267438828943 +fresh-en-2,8,9,K,all_heads,400,0.16511117666959763,0.24270714074373245,0.34380056560039524,0.2489963484928012,400,0.6807672202587128,0.9749633073806763,1.2984531521797182,0.9798759507387876 +fresh-en-2,8,9,V,weak_half,400,0.04923901222646236,0.09516975656151772,0.17818013429641727,0.107487897798419,400,0.14183374345302582,0.2558494955301285,0.4267560184001923,0.27771912729367615 +fresh-en-2,8,9,V,strong_half,400,0.09868927299976349,0.18543319404125214,0.3130709171295166,0.1992524154484272,400,0.32738474309444426,0.5321256816387177,0.903970205783844,0.5869241042062641 +fresh-en-2,8,9,V,random_half,400,0.0709797389805317,0.1359727531671524,0.2658272802829743,0.1557344006933272,400,0.22334586083889008,0.39216992259025574,0.6533923864364624,0.4273674661666155 +fresh-en-2,8,9,V,all_heads,400,0.09600097239017487,0.19496743381023407,0.35399151742458346,0.2149423841200769,400,0.48316113948822026,0.7940475642681122,1.3342432498931884,0.8646432314813137 +fresh-en-2,8,9,QKV,weak_half,400,0.0755506120622158,0.1162651814520359,0.16556834876537327,0.11954979645088315,400,0.21265350729227067,0.31769615411758423,0.42879785001277926,0.32028101541101933 +fresh-en-2,8,9,QKV,strong_half,400,0.17130101174116136,0.2283269539475441,0.31501202583312987,0.23769124707207084,400,0.5340533077716827,0.6914201378822327,0.8608787596225739,0.6989980760961771 +fresh-en-2,8,9,QKV,random_half,400,0.12656363248825073,0.17972997575998306,0.2607716947793961,0.18919972244650127,400,0.3562931627035141,0.5017023682594299,0.6564644396305084,0.5064664828777313 +fresh-en-2,8,9,QKV,all_heads,400,0.18131361156702042,0.25411149859428406,0.3471276700496674,0.2598065463453531,400,0.7645834684371948,1.0215259194374084,1.274821150302887,1.01927909091115 +fresh-en-2,9,10,Q,weak_half,400,0.06260908544063568,0.1049736812710762,0.18661284595727923,0.1158860001526773,400,0.16759291887283326,0.2877777963876724,0.46822879910469056,0.30347544454038144 +fresh-en-2,9,10,Q,strong_half,400,0.12650530338287352,0.22236952185630798,0.39169742763042464,0.25630814999341966,400,0.3935243934392929,0.6456329226493835,1.0217528820037842,0.6868917352333664 +fresh-en-2,9,10,Q,random_half,400,0.09737184271216393,0.16510310024023056,0.2872407615184784,0.18072387859225272,400,0.2770599484443665,0.44669002294540405,0.7741307854652405,0.4892986752837896 +fresh-en-2,9,10,Q,all_heads,400,0.14727880507707597,0.24678459763526917,0.4451851129531861,0.28828183831647036,400,0.5841196775436401,0.940103143453598,1.4729133367538458,0.9903671786934137 +fresh-en-2,9,10,K,weak_half,400,0.07546543776988983,0.11805541440844536,0.1874848484992981,0.12504147686064243,400,0.193598173558712,0.28334319591522217,0.4127568542957308,0.29855245269834996 +fresh-en-2,9,10,K,strong_half,400,0.1348200634121895,0.2116714045405388,0.3040048748254776,0.2198746239207685,400,0.4150237739086151,0.5772019624710083,0.8282717645168305,0.5962334944307804 +fresh-en-2,9,10,K,random_half,400,0.11266338378190995,0.17139719426631927,0.26648482382297517,0.18223484195768833,400,0.29520902037620544,0.43547025322914124,0.633318364620209,0.4457104074209928 +fresh-en-2,9,10,K,all_heads,400,0.16042915284633635,0.23898569494485855,0.36059972941875473,0.25203531682491304,400,0.6156108260154725,0.8619667291641235,1.2304174184799195,0.8947859485447407 +fresh-en-2,9,10,V,weak_half,400,0.04554114378988743,0.0843483954668045,0.17898856550455094,0.10278829288203269,400,0.13397155553102494,0.21989019215106964,0.39537259340286257,0.24747534077614547 +fresh-en-2,9,10,V,strong_half,400,0.08402921929955483,0.14943302422761917,0.2845828354358673,0.1688336451910436,400,0.26927646100521085,0.4289074093103409,0.7417753636837008,0.47413283716887233 +fresh-en-2,9,10,V,random_half,400,0.06750376299023628,0.12419060990214348,0.23469255417585375,0.13858943532221019,400,0.19697765707969667,0.31682009994983673,0.581931507587433,0.36057353792712094 +fresh-en-2,9,10,V,all_heads,400,0.10437864512205124,0.17656106501817703,0.33849772512912757,0.20263156364671886,400,0.4167629569768906,0.6541136503219604,1.147709023952484,0.7216081768274307 +fresh-en-2,9,10,QKV,weak_half,400,0.07167848944664001,0.11324717104434967,0.17674364745616913,0.11920010052621365,400,0.189215549826622,0.2884126305580139,0.41544484496116646,0.2966003199853003 +fresh-en-2,9,10,QKV,strong_half,400,0.14228248447179795,0.2114020511507988,0.3148634314537049,0.22813525589182973,400,0.4236171096563339,0.6046692132949829,0.8341619729995731,0.6276269568502904 +fresh-en-2,9,10,QKV,random_half,400,0.11186779439449311,0.17063499987125397,0.24672154486179357,0.17522302735596895,400,0.2985461831092835,0.44271403551101685,0.6298513889312746,0.45768570382148027 +fresh-en-2,9,10,QKV,all_heads,400,0.16594154834747316,0.23964254558086395,0.36320458352565765,0.2614496209472418,400,0.6403672754764557,0.9104771614074707,1.2211272120475771,0.9242272755503654 +fresh-en-2,10,11,Q,weak_half,200,0.07038184031844139,0.10132408887147903,0.13709059357643127,0.10182490890845657,200,0.18107707798480988,0.26173196732997894,0.3545970439910889,0.26413913879543544 +fresh-en-2,10,11,Q,strong_half,200,0.1719125911593437,0.2206062451004982,0.27472214996814726,0.22130272820591926,200,0.4388366162776947,0.5981227159500122,0.7494932651519776,0.5971695150434971 +fresh-en-2,10,11,Q,random_half,200,0.11728009507060051,0.16756664961576462,0.22152178734540937,0.16910921260714531,200,0.2951887726783753,0.4199138283729553,0.5603043735027313,0.4263012070953846 +fresh-en-2,10,11,Q,all_heads,200,0.20134070217609407,0.2543593943119049,0.3090263783931732,0.25503547973930835,200,0.6217361330986023,0.8588953614234924,1.0879042029380799,0.8613086573779583 +fresh-en-2,10,11,K,weak_half,200,0.05850331857800484,0.08487061411142349,0.11172247529029843,0.0837748327292502,200,0.14990698397159577,0.2158988118171692,0.292885884642601,0.21610772892832755 +fresh-en-2,10,11,K,strong_half,200,0.21238570958375935,0.25445112586021423,0.3248644828796386,0.2623239817470312,200,0.4847216218709946,0.6385707855224609,0.7794338881969451,0.634505704343319 +fresh-en-2,10,11,K,random_half,200,0.11164127811789513,0.1682373508810997,0.2751823604106903,0.18038799796253444,200,0.27234992384910583,0.39897817373275757,0.5764695227146148,0.41511692173779013 +fresh-en-2,10,11,K,all_heads,200,0.22468682527542114,0.2773387134075165,0.346183007955551,0.28219417095184324,200,0.6621845543384552,0.8579957485198975,1.042912435531616,0.8506134343147278 +fresh-en-2,10,11,V,weak_half,200,0.08351906388998032,0.11816345155239105,0.14616515785455703,0.11726409927010537,200,0.21933752298355103,0.30671632289886475,0.40581075251102444,0.311245681270957 +fresh-en-2,10,11,V,strong_half,200,0.20251600444316864,0.23767173290252686,0.29717555046081545,0.24701301991939545,200,0.5424065947532654,0.670983076095581,0.8581440567970275,0.6866811785101891 +fresh-en-2,10,11,V,random_half,200,0.14061032831668854,0.18410652875900269,0.2515390485525131,0.18883809573948385,200,0.35138703882694244,0.478024423122406,0.6620442509651184,0.49503570079803466 +fresh-en-2,10,11,V,all_heads,200,0.2246594160795212,0.26335592567920685,0.3278277516365051,0.2696025139093399,200,0.7910944104194642,0.9786365032196045,1.2328994631767272,0.9979268705844879 +fresh-en-2,10,11,QKV,weak_half,200,0.07423313781619072,0.10163884237408638,0.12991934567689895,0.10231441101059317,200,0.1899012416601181,0.2680436968803406,0.3463482290506363,0.26946202233433725 +fresh-en-2,10,11,QKV,strong_half,200,0.2063558727502823,0.24629150331020355,0.2853017061948776,0.24610054023563863,200,0.5239730596542358,0.6595558226108551,0.8016687393188476,0.6603868694603443 +fresh-en-2,10,11,QKV,random_half,200,0.1312642976641655,0.1799336075782776,0.23550689071416855,0.18195221915841103,200,0.3162532776594162,0.44466981291770935,0.6025232434272766,0.45797576889395714 +fresh-en-2,10,11,QKV,all_heads,200,0.23327601402997972,0.2708330899477005,0.3110144823789597,0.2710502111911774,200,0.7359493970870972,0.9326356649398804,1.1458848476409913,0.929848895072937 +fresh-en-2,10,11,JS,weak_half,200,0.0009476990089751781,0.002315948950126767,0.005958323320373891,0.002919635430152994,200,0.0013236737577244642,0.0029417811892926693,0.006184877408668399,0.0034075136606406885 +fresh-en-2,10,11,JS,strong_half,200,0.010464109852910042,0.024222557432949543,0.05165547169744967,0.030114642097614705,200,0.009930481668561697,0.019933870062232018,0.03750751726329326,0.0242697114427574 +fresh-en-2,10,11,JS,random_half,200,0.005059097008779646,0.013714892789721489,0.03391791358590126,0.017332479990436696,200,0.003415789781138301,0.009449750650674105,0.026962711848318572,0.014601738541387022 +fresh-en-2,10,11,JS,all_heads,200,0.011890527978539468,0.030723503790795803,0.06180024854838847,0.03625898967729881,200,0.011709014978259803,0.023373059928417206,0.04310774952173232,0.027677225096849726 +fresh-en-2,10,11,sqrtJS,weak_half,200,0.03078422099351883,0.04812430776655674,0.0771872766315937,0.050625629164278506,200,0.09719870537519457,0.14401781558990479,0.20405741930007934,0.14747192542999982 +fresh-en-2,10,11,sqrtJS,strong_half,200,0.10229422003030778,0.15563491731882095,0.22726870030164714,0.16295818198472262,200,0.2517652750015259,0.35555365681648254,0.48079841434955595,0.35950615227222443 +fresh-en-2,10,11,sqrtJS,random_half,200,0.07112659886479378,0.11711060255765915,0.1841680958867073,0.1218267538305372,200,0.14671407639980316,0.23617949336767197,0.3777141749858856,0.2521909896656871 +fresh-en-2,10,11,sqrtJS,all_heads,200,0.10904362872242927,0.17528118938207626,0.24859247654676433,0.18002940978854895,200,0.358953469991684,0.4935784339904785,0.6759456992149353,0.5069780769944191 +fresh-en-2,11,12,Q,weak_half,400,0.09977171197533607,0.18265271186828613,0.332904675602913,0.2022944123018533,400,0.21803484410047533,0.3230695426464081,0.5132083177566529,0.34421282082796095 +fresh-en-2,11,12,Q,strong_half,400,0.11882370710372925,0.18954575806856155,0.32980991899967194,0.2070493898168206,400,0.4305072516202927,0.7114525437355042,1.2002669811248778,0.7676704281568527 +fresh-en-2,11,12,Q,random_half,400,0.11389405429363252,0.20444241166114807,0.37898698151111604,0.23368638389743865,400,0.30692842304706575,0.514281302690506,0.8917958378791809,0.5574041147902608 +fresh-en-2,11,12,Q,all_heads,400,0.11444473043084145,0.1689644306898117,0.2740604877471924,0.18405889308080078,400,0.6585037589073182,1.0366052985191345,1.6983477354049685,1.1118832495808602 +fresh-en-2,11,12,K,weak_half,400,0.08373332992196084,0.14872167259454727,0.27649546861648566,0.16443962642922996,400,0.16544622331857683,0.25857363641262054,0.4130991369485856,0.27776868361979723 +fresh-en-2,11,12,K,strong_half,400,0.09545204862952232,0.16010010987520218,0.26359431445598613,0.17447322363965212,400,0.3165766060352325,0.5074418783187866,0.9016786456108095,0.5732645041495562 +fresh-en-2,11,12,K,random_half,400,0.10239675268530846,0.18285255134105682,0.3400317251682282,0.2111460941284895,400,0.21866856068372725,0.3835843503475189,0.6437533855438234,0.42313692063093183 +fresh-en-2,11,12,K,all_heads,400,0.10049738213419915,0.1485031694173813,0.2513172686100006,0.1650553133059293,400,0.4983573883771897,0.762269914150238,1.304659652709961,0.8510331868380309 +fresh-en-2,11,12,V,weak_half,400,0.06728086397051812,0.13117065280675888,0.2587785035371781,0.14702996608801186,400,0.13126679360866547,0.23100411146879196,0.38090438246726993,0.250036850720644 +fresh-en-2,11,12,V,strong_half,400,0.06807193458080292,0.1297636479139328,0.25808417201042183,0.1487276737112552,400,0.23939171433448792,0.45916634798049927,0.9209554910659791,0.5343375466763973 +fresh-en-2,11,12,V,random_half,400,0.06998846530914307,0.14848899096250534,0.3256516069173813,0.18188232609070837,400,0.176853309571743,0.33676673471927643,0.6636181771755219,0.3898086488246918 +fresh-en-2,11,12,V,all_heads,400,0.07481089606881142,0.13868942111730576,0.2521002501249314,0.15484060080721973,400,0.3862847894430161,0.6913402378559113,1.2980188488960267,0.7843743977323174 +fresh-en-2,11,12,QKV,weak_half,400,0.09860116541385651,0.1673557162284851,0.27665888965129853,0.17937433417886495,400,0.1952506631612778,0.29007770121097565,0.42550180852413183,0.30392378512769935 +fresh-en-2,11,12,QKV,strong_half,400,0.10824550613760948,0.17452948540449142,0.27175362408161163,0.18479891059920192,400,0.3949659109115601,0.6348735690116882,0.9616786599159245,0.6611970131844283 +fresh-en-2,11,12,QKV,random_half,400,0.11043949723243714,0.20388830453157425,0.3385278552770615,0.22008628140203654,400,0.27744942009449003,0.44727206230163574,0.7476902186870575,0.48183888792991636 +fresh-en-2,11,12,QKV,all_heads,400,0.11405921503901481,0.16539092361927032,0.24534664154052735,0.1741924537345767,400,0.596085786819458,0.9248353838920593,1.3828170061111453,0.9651207979768515 +fresh-en-2,12,13,Q,weak_half,400,0.049296176806092266,0.0800744779407978,0.12034962624311449,0.08362000953406096,400,0.14770948886871338,0.23183221369981766,0.3393673032522202,0.23954888643696903 +fresh-en-2,12,13,Q,strong_half,400,0.11290597915649414,0.18300053477287292,0.3227073967456818,0.20284499370492995,400,0.3434539705514908,0.5574801862239838,0.8376962959766387,0.5744885654002428 +fresh-en-2,12,13,Q,random_half,400,0.08157389387488366,0.13830771297216415,0.23317327797412873,0.15038773416541518,400,0.24880006015300754,0.3794826865196228,0.5895817995071413,0.40710232853889466 +fresh-en-2,12,13,Q,all_heads,400,0.12027851641178132,0.1984739601612091,0.3152281761169434,0.21180270349606872,400,0.514024305343628,0.7920748889446259,1.1551995754241944,0.8140374537557363 +fresh-en-2,12,13,K,weak_half,400,0.05065112300217152,0.085109643638134,0.13974934369325637,0.09146209066733718,400,0.13828784227371216,0.218926802277565,0.3425118297338486,0.23135700017213823 +fresh-en-2,12,13,K,strong_half,400,0.09996561482548713,0.1640303060412407,0.24728863835334777,0.17079760933294894,400,0.30333846509456636,0.4575664550065994,0.627351772785187,0.46979754336178303 +fresh-en-2,12,13,K,random_half,400,0.08511418476700783,0.1347767859697342,0.201189586520195,0.13963640212081374,400,0.22017076462507248,0.33563297986984253,0.4896285802125931,0.34879403820261357 +fresh-en-2,12,13,K,all_heads,400,0.11032596975564957,0.1828026995062828,0.2699695616960526,0.18717810241505503,400,0.44422444701194763,0.6919434070587158,0.9595236122608185,0.7011545442044735 +fresh-en-2,12,13,V,weak_half,400,0.03873490691184998,0.06543773785233498,0.12210180163383488,0.07407324600033462,400,0.10404559150338173,0.17661771923303604,0.26930986344814306,0.18586777025833726 +fresh-en-2,12,13,V,strong_half,400,0.07261145859956741,0.12703649699687958,0.2096351683139801,0.13778579558711498,400,0.24554488956928253,0.37312333285808563,0.5433411300182345,0.39053594689816234 +fresh-en-2,12,13,V,random_half,400,0.059356636554002765,0.10730038583278656,0.17845096886157993,0.11300956767983734,400,0.17071987688541412,0.2666172534227371,0.4276963233947754,0.28492866246029735 +fresh-en-2,12,13,V,all_heads,400,0.08505288809537888,0.14563308656215668,0.2388486757874489,0.15437128857709467,400,0.37649413645267493,0.5413700044155121,0.7962691307067872,0.576403716430068 +fresh-en-2,12,13,QKV,weak_half,400,0.05754005126655102,0.08169753476977348,0.12612495571374893,0.08588925839401781,400,0.15580129325389863,0.22030457854270935,0.3149178594350815,0.22834967747330664 +fresh-en-2,12,13,QKV,strong_half,400,0.12210273519158364,0.17147570848464966,0.24513592571020126,0.17937609277665614,400,0.37110573053359985,0.5085664391517639,0.6524723291397095,0.5100879909098148 +fresh-en-2,12,13,QKV,random_half,400,0.09477518051862717,0.13514243811368942,0.18772287815809252,0.14040848593227565,400,0.25469406247138976,0.3600640892982483,0.4987926840782166,0.36811968989670274 +fresh-en-2,12,13,QKV,all_heads,400,0.13309793025255204,0.18979545682668686,0.26132586002349856,0.19240224976092576,400,0.5387952864170075,0.7266457378864288,0.9496304214000703,0.7384376665949821 +fresh-en-2,13,14,Q,weak_half,400,0.03736812211573124,0.07053525745868683,0.12779943346977235,0.07728623909177258,400,0.10172585174441338,0.1879143938422203,0.316551411151886,0.20142320394515992 +fresh-en-2,13,14,Q,strong_half,400,0.1028013601899147,0.17265401035547256,0.27668344676494605,0.18059237501118333,400,0.28165948390960693,0.5016399323940277,0.8042557239532471,0.5186002370156347 +fresh-en-2,13,14,Q,random_half,400,0.06843169629573823,0.1272887960076332,0.21648157089948655,0.13666861523408444,400,0.1909718066453934,0.33945387601852417,0.5598009586334229,0.36050841158255936 +fresh-en-2,13,14,Q,all_heads,400,0.11675868630409242,0.19389842450618744,0.30167636275291443,0.20051220152061433,400,0.4204987436532974,0.6985971927642822,1.0954076528549195,0.720023437179625 +fresh-en-2,13,14,K,weak_half,400,0.04492225535213947,0.07910692691802979,0.1298276796936989,0.08348192829173058,400,0.11259646788239479,0.19223491102457047,0.3066891551017762,0.20397487811744214 +fresh-en-2,13,14,K,strong_half,400,0.11873571574687958,0.17922590672969818,0.2751345276832581,0.1900436809193343,400,0.3405205368995667,0.4974425882101059,0.7407350957393648,0.5293476174026728 +fresh-en-2,13,14,K,random_half,400,0.09162245318293571,0.14372211694717407,0.2482945784926415,0.15922621226869524,400,0.218488647043705,0.34812790155410767,0.5383987307548523,0.36836636867374184 +fresh-en-2,13,14,K,all_heads,400,0.13042968958616258,0.18927574902772903,0.2941136062145233,0.20254017930477858,400,0.4758277446031571,0.682720273733139,1.0576205968856813,0.733322496637702 +fresh-en-2,13,14,V,weak_half,400,0.029407133162021638,0.058150285854935646,0.14237823486328124,0.07158113559242338,400,0.07562845200300217,0.14432056993246078,0.31114279031753556,0.17628843011334538 +fresh-en-2,13,14,V,strong_half,400,0.0786085106432438,0.13082808256149292,0.28467754423618324,0.16275794556364417,400,0.2183425173163414,0.3842757046222687,0.7525210559368134,0.44498509060591457 +fresh-en-2,13,14,V,random_half,400,0.054213215783238414,0.09714895859360695,0.24903912842273715,0.13135469743050635,400,0.14933510720729828,0.26857903599739075,0.5468106210231782,0.31065571364015343 +fresh-en-2,13,14,V,all_heads,400,0.08647480010986329,0.14658238738775253,0.305269193649292,0.17669258575886487,400,0.3086221396923065,0.5312029421329498,1.021984076499939,0.62127352014184 +fresh-en-2,13,14,QKV,weak_half,400,0.04519356153905392,0.0740337036550045,0.12168385535478593,0.08075229680165648,400,0.11466182768344879,0.18871770054101944,0.30689706504344944,0.2037953567504883 +fresh-en-2,13,14,QKV,strong_half,400,0.11820140108466148,0.17281553894281387,0.26743829846382144,0.18486969160847366,400,0.34954228699207307,0.4950675219297409,0.7563082456588746,0.5294560895115137 +fresh-en-2,13,14,QKV,random_half,400,0.08700612410902978,0.13835196197032928,0.22357573509216308,0.14970461273565888,400,0.21796901673078536,0.34942108392715454,0.5314964056015016,0.3674422583729029 +fresh-en-2,13,14,QKV,all_heads,400,0.12950581908226014,0.18701691180467606,0.2851172953844071,0.20059745650738478,400,0.4834583580493927,0.6776576638221741,1.0683794379234317,0.7332514478266239 +fresh-en-2,14,15,Q,weak_half,200,0.0636510193347931,0.09577921032905579,0.13068428933620452,0.09838483411818742,200,0.1664366230368614,0.24635696411132812,0.3448635399341583,0.2537464252486825 +fresh-en-2,14,15,Q,strong_half,200,0.20775139927864075,0.2646036446094513,0.3381988435983658,0.26955482870340347,200,0.5208715200424194,0.6510580778121948,0.8127166271209717,0.66664207264781 +fresh-en-2,14,15,Q,random_half,200,0.12492759004235268,0.18270689994096756,0.25438238978385924,0.18908626589924096,200,0.3133884102106094,0.44208599627017975,0.6107684314250945,0.45309692092239856 +fresh-en-2,14,15,Q,all_heads,200,0.21830895990133287,0.2753938287496567,0.3441651105880737,0.2794119719415903,200,0.7049972534179687,0.889905571937561,1.1669517993927,0.9203884989023209 +fresh-en-2,14,15,K,weak_half,200,0.035423438996076584,0.048680853098630905,0.06800558418035507,0.051437660083174706,200,0.08549972772598267,0.1174682080745697,0.17463333904743195,0.1247058668732643 +fresh-en-2,14,15,K,strong_half,200,0.14310666173696518,0.22970852255821228,0.30378997325897217,0.22599894620478153,200,0.34101538062095643,0.5174771547317505,0.7135810375213623,0.5214048288762569 +fresh-en-2,14,15,K,random_half,200,0.08473309278488159,0.144621841609478,0.24857259094715117,0.15614630311727523,200,0.1904165729880333,0.2994358241558075,0.4798284202814102,0.3214251428470016 +fresh-en-2,14,15,K,all_heads,200,0.15464893430471421,0.23393215984106064,0.3139076441526413,0.23495981276035308,200,0.4545753985643387,0.6414374113082886,0.8653814673423766,0.6461106944084167 +fresh-en-2,14,15,V,weak_half,200,0.03550384193658829,0.05590137839317322,0.07936130464076996,0.05739534032531082,200,0.09269408881664276,0.1352875530719757,0.19336654841899872,0.1398423895984888 +fresh-en-2,14,15,V,strong_half,200,0.13014940321445465,0.18172790110111237,0.2705060839653015,0.19236074794083835,200,0.318980222940445,0.4411599934101105,0.6198874115943909,0.4580790691077709 +fresh-en-2,14,15,V,random_half,200,0.07437047734856606,0.12122226133942604,0.2206067830324173,0.13595697689801456,200,0.17526627331972125,0.2790182828903198,0.4682885408401489,0.2961802526190877 +fresh-en-2,14,15,V,all_heads,200,0.1440908119082451,0.19837813079357147,0.27883152365684505,0.20440662756562233,200,0.43765890598297125,0.5745079815387726,0.820201027393341,0.5979214584827424 +fresh-en-2,14,15,QKV,weak_half,200,0.051787834614515305,0.0715431496500969,0.09600503072142601,0.07293996496126055,200,0.12968271672725679,0.18165204674005508,0.24778558015823363,0.1866216842457652 +fresh-en-2,14,15,QKV,strong_half,200,0.2041396364569664,0.23789264261722565,0.2699014127254486,0.2370678836107254,200,0.5151191651821136,0.5987609922885895,0.6814356684684754,0.5962325042486191 +fresh-en-2,14,15,QKV,random_half,200,0.11204893738031388,0.17136747390031815,0.21706339865922927,0.16766387581825257,200,0.2720547378063202,0.38748225569725037,0.49729853868484497,0.38765632070600986 +fresh-en-2,14,15,QKV,all_heads,200,0.2157004415988922,0.24718062579631805,0.28046256601810454,0.24671753399074078,200,0.6537230670452118,0.7794557511806488,0.9111264288425446,0.7828541907668114 +fresh-en-2,14,15,JS,weak_half,200,0.000566274079028517,0.002160659059882164,0.0063986475579440585,0.0037506354102515614,200,0.0011499425396323205,0.0026964331045746803,0.006353971594944595,0.003389898348686984 +fresh-en-2,14,15,JS,strong_half,200,0.0070406328421086075,0.01986111421138048,0.041440534964203836,0.02287774618540425,200,0.012600267492234708,0.02380199171602726,0.04955219477415084,0.028348808344453573 +fresh-en-2,14,15,JS,random_half,200,0.004474284127354622,0.011049839667975903,0.0340390160679817,0.015738951050734612,200,0.004855869244784117,0.0122651606798172,0.03032874707132578,0.01588385404087603 +fresh-en-2,14,15,JS,all_heads,200,0.006954804062843323,0.019897349178791046,0.04299733601510524,0.023360251858830453,200,0.014904378727078438,0.026413628831505775,0.054044562950730325,0.031738706710748375 +fresh-en-2,14,15,sqrtJS,weak_half,200,0.023796430975198745,0.046482134610414505,0.07999045327305793,0.05244437991175801,200,0.08159168139100074,0.1308000087738037,0.19865619391202927,0.135908197555691 +fresh-en-2,14,15,sqrtJS,strong_half,200,0.08390834778547288,0.14092885702848434,0.2035694882273674,0.14341442342847585,200,0.28551762700080874,0.3806847780942917,0.542874550819397,0.39679291278123857 +fresh-en-2,14,15,sqrtJS,random_half,200,0.06688877046108246,0.10511806607246399,0.18448149710893627,0.11662591470405459,200,0.16636089831590653,0.2510561943054199,0.382892045378685,0.2626064766570926 +fresh-en-2,14,15,sqrtJS,all_heads,200,0.0833953082561493,0.14105772227048874,0.20735650062561034,0.14499781841412188,200,0.38702398240566255,0.5066772699356079,0.707418382167816,0.5327011090517044 +fresh-en-2,15,16,Q,weak_half,400,0.05123864337801933,0.11455101147294044,0.19551145136356354,0.11863171414472162,400,0.09346392825245857,0.20022311061620712,0.30094145834445957,0.19985113121569156 +fresh-en-2,15,16,Q,strong_half,400,0.0598639078438282,0.15667196363210678,0.26432085037231445,0.16155789588112385,400,0.13466916531324388,0.33022043108940125,0.6219455361366273,0.35966296633705497 +fresh-en-2,15,16,Q,random_half,400,0.05961190536618233,0.1369868591427803,0.2507007628679276,0.14897025751881302,400,0.11907244399189951,0.2537682354450226,0.47911027073860174,0.2794033911265433 +fresh-en-2,15,16,Q,all_heads,400,0.10157701075077058,0.22258032113313675,0.3503028482198716,0.2271798975393176,400,0.23205759674310686,0.5322947204113007,0.9188684761524202,0.5595140976458788 +fresh-en-2,15,16,K,weak_half,400,0.07135206684470177,0.1161087304353714,0.1981597542762757,0.1272178131621331,400,0.12825906425714492,0.18338917195796967,0.2736615598201752,0.19221443923190237 +fresh-en-2,15,16,K,strong_half,400,0.09580065906047822,0.15102573484182358,0.22243296951055533,0.15715362882241607,400,0.2141096457839012,0.3098422437906265,0.5001795560121537,0.3388399091362953 +fresh-en-2,15,16,K,random_half,400,0.08844384476542473,0.1433994323015213,0.250051948428154,0.15978431965224446,400,0.16344715803861617,0.24507273733615875,0.3972939014434815,0.265891722869128 +fresh-en-2,15,16,K,all_heads,400,0.14998015612363816,0.21934082359075546,0.321985399723053,0.2276856290921569,400,0.34262027144432067,0.4971429109573364,0.7807130098342896,0.5310543486475945 +fresh-en-2,15,16,V,weak_half,400,0.06580682471394539,0.10739576444029808,0.18363135457038882,0.11649420568719507,400,0.11299866363406182,0.16636886447668076,0.24902693629264835,0.17495502792298795 +fresh-en-2,15,16,V,strong_half,400,0.09722905158996582,0.1558762714266777,0.2252307325601578,0.15947166584432126,400,0.2065848782658577,0.29037706553936005,0.47198382019996643,0.31544845417141915 +fresh-en-2,15,16,V,random_half,400,0.08231866359710693,0.1321501061320305,0.235815766453743,0.14807749326340855,400,0.15424002707004547,0.22303663194179535,0.36032584011554725,0.24385395638644694 +fresh-en-2,15,16,V,all_heads,400,0.16220051646232606,0.22136453539133072,0.30078548789024356,0.2287060561403632,400,0.3260258764028549,0.46211689710617065,0.7086701989173889,0.4904034804552794 +fresh-en-2,15,16,QKV,weak_half,400,0.07482635751366615,0.11814026534557343,0.18720453679561616,0.1243088761717081,400,0.13402340561151505,0.18736542761325836,0.26739251613616943,0.19553621884435415 +fresh-en-2,15,16,QKV,strong_half,400,0.10464766249060631,0.16050666570663452,0.2225837901234627,0.16450546000152827,400,0.23496161103248597,0.32092341780662537,0.5241274535655976,0.35315464187413453 +fresh-en-2,15,16,QKV,random_half,400,0.09668755009770394,0.14659055322408676,0.2389873579144478,0.15789923651143908,400,0.17605603933334352,0.24985555559396744,0.39795373082160956,0.2738386351428926 +fresh-en-2,15,16,QKV,all_heads,400,0.17091473042964936,0.23185046762228012,0.3048892825841904,0.23399994557723403,400,0.3720066010951996,0.5081002116203308,0.7878897070884705,0.5486908616870642 +fresh-en-2,16,17,Q,weak_half,400,0.012656503077596427,0.03420771472156048,0.06629591956734657,0.036833753340179104,400,0.034769652783870696,0.09611674398183823,0.17679314315319064,0.10048837738577276 +fresh-en-2,16,17,Q,strong_half,400,0.025776355154812337,0.08325633406639099,0.16797593086957935,0.09064867308363318,400,0.0719117671251297,0.21125388145446777,0.41614089906215673,0.23124037547037005 +fresh-en-2,16,17,Q,random_half,400,0.019699856266379358,0.060373580083251,0.11815553829073906,0.06541101089678705,400,0.05304737500846386,0.15621069818735123,0.2948747009038925,0.16508588681463152 +fresh-en-2,16,17,Q,all_heads,400,0.02832107860594988,0.0913516953587532,0.18562427610158921,0.09897243125829845,400,0.10853456184267998,0.3044695109128952,0.581692159175873,0.3317287537455559 +fresh-en-2,16,17,K,weak_half,400,0.035717704147100446,0.05485176295042038,0.08573992252349853,0.058218489848077294,400,0.09903450012207031,0.13992224633693695,0.1983544945716858,0.144892491530627 +fresh-en-2,16,17,K,strong_half,400,0.07700884565711022,0.1128385066986084,0.15739901065826417,0.11607947569340467,400,0.22733717113733293,0.32590754330158234,0.46119292676448825,0.33631990786641836 +fresh-en-2,16,17,K,random_half,400,0.0580638624727726,0.08820495009422302,0.12948624789714813,0.09178304217755795,400,0.15568237602710724,0.223580501973629,0.32229941487312325,0.23622647043317557 +fresh-en-2,16,17,K,all_heads,400,0.08490124344825745,0.12312906607985497,0.175516214966774,0.1268406833615154,400,0.3311691224575043,0.4691314399242401,0.6506060600280762,0.481212399341166 +fresh-en-2,16,17,V,weak_half,400,0.029775776155292987,0.049317045137286186,0.0770153246819973,0.05241472051478922,400,0.08034836202859878,0.12420940026640892,0.18770878016948703,0.12985253293067217 +fresh-en-2,16,17,V,strong_half,400,0.06816741898655891,0.10631216317415237,0.17547938078641895,0.11573295194655657,400,0.2104615554213524,0.29943709075450897,0.440213143825531,0.31355204217135907 +fresh-en-2,16,17,V,random_half,400,0.04508389085531235,0.07772915065288544,0.13086697161197663,0.08569066641852259,400,0.13533741384744644,0.211038738489151,0.30887264907360085,0.21953507950529455 +fresh-en-2,16,17,V,all_heads,400,0.07796342968940735,0.12007970362901688,0.19316766411066055,0.12890729457139968,400,0.28255532383918763,0.4233954846858978,0.6171590805053712,0.4434045723080635 +fresh-en-2,16,17,QKV,weak_half,400,0.03562949039041996,0.0505797453224659,0.06894291564822197,0.052061988594941794,400,0.09724994078278541,0.12963584810495377,0.17383714765310287,0.13287677466869355 +fresh-en-2,16,17,QKV,strong_half,400,0.07809429839253426,0.10800384730100632,0.15669801384210588,0.11328532329760492,400,0.22656388580799106,0.3008946478366852,0.41897636353969575,0.314148297123611 +fresh-en-2,16,17,QKV,random_half,400,0.0565076507627964,0.08160967007279396,0.12170791402459145,0.08574995946139097,400,0.15573347061872483,0.21460937708616257,0.3045534282922745,0.22085205713286996 +fresh-en-2,16,17,QKV,all_heads,400,0.08839923590421678,0.11997655406594276,0.16460000574588776,0.12465949300676585,400,0.32350266575813297,0.4299907237291336,0.5844420313835145,0.4470250725001097 +fresh-en-2,17,18,Q,weak_half,400,0.0110936364158988,0.03049679845571518,0.053183586522936824,0.031163888026494533,400,0.031531427055597305,0.08788643404841423,0.14158959686756134,0.08621070317924023 +fresh-en-2,17,18,Q,strong_half,400,0.034567391872406004,0.10147760063409805,0.16429675817489625,0.10055154041852803,400,0.0996741570532322,0.26477058231830597,0.4369262009859086,0.2643332630954683 +fresh-en-2,17,18,Q,random_half,400,0.020875290408730512,0.06747136637568474,0.13404856622219088,0.07395591175882146,400,0.058083542808890345,0.16366812586784363,0.30481848716735843,0.17330203636083752 +fresh-en-2,17,18,Q,all_heads,400,0.03544425815343857,0.10903982073068619,0.16895652562379837,0.10548504761885852,400,0.13763022422790527,0.3529496192932129,0.5568669795989992,0.3505439651198685 +fresh-en-2,17,18,K,weak_half,400,0.015546692721545696,0.02775663696229458,0.042815745621919636,0.029032475519925355,400,0.04602168053388596,0.07744399458169937,0.1090365529060364,0.07816219703294337 +fresh-en-2,17,18,K,strong_half,400,0.05567496232688427,0.08777879178524017,0.12522005289793017,0.09047897231765091,400,0.15193795263767243,0.2156439945101738,0.29991729855537413,0.2214925553277135 +fresh-en-2,17,18,K,random_half,400,0.034847008809447286,0.058312173932790756,0.09820132553577424,0.0636365364678204,400,0.08689631745219231,0.1433395892381668,0.20807379782199864,0.14782599083147943 +fresh-en-2,17,18,K,all_heads,400,0.05846467912197113,0.09141025692224503,0.13771509379148486,0.09527765364386141,400,0.19883154332637787,0.295414075255394,0.40346068739891056,0.29965475283563137 +fresh-en-2,17,18,V,weak_half,400,0.018144098855555058,0.028229237534105778,0.04013628736138345,0.029082243428565562,400,0.05367904119193554,0.07898611202836037,0.10664059147238732,0.07972229118924588 +fresh-en-2,17,18,V,strong_half,400,0.09353125244379043,0.12732776254415512,0.16738848984241486,0.12778125851415098,400,0.22088093906641007,0.2822072058916092,0.3517431527376177,0.2839921898953617 +fresh-en-2,17,18,V,random_half,400,0.038611609116196634,0.07429002225399017,0.14245420098304748,0.08247321377508342,400,0.09842480570077897,0.17073392122983932,0.2676084578037262,0.1777236928232014 +fresh-en-2,17,18,V,all_heads,400,0.09588091596961021,0.12904494255781174,0.17209460586309433,0.13238918744027614,400,0.28444458544254303,0.3630967140197754,0.45597008764743807,0.363714482113719 +fresh-en-2,17,18,QKV,weak_half,400,0.02091757133603096,0.030559608712792397,0.04145057760179043,0.031135324486531318,400,0.05725748836994171,0.0847792997956276,0.1124622292816639,0.08549444738775491 +fresh-en-2,17,18,QKV,strong_half,400,0.08763744458556175,0.11027907952666283,0.14007240682840347,0.1121645924448967,400,0.20907141268253326,0.2649668753147125,0.3405623197555543,0.27302359998226167 +fresh-en-2,17,18,QKV,random_half,400,0.04363566339015961,0.07664044201374054,0.11408795788884163,0.0780296454206109,400,0.10895067900419235,0.17365123331546783,0.24718101620674132,0.17628412645310162 +fresh-en-2,17,18,QKV,all_heads,400,0.09268839433789253,0.1135963499546051,0.14575880467891694,0.1171895575709641,400,0.2726540297269821,0.3497917205095291,0.4507372379302979,0.3585180474817753 +fresh-en-2,18,19,Q,weak_half,200,0.04991522654891014,0.06345029547810555,0.07465114593505859,0.06271788576617837,200,0.1317598417401314,0.16468250006437302,0.1921829268336296,0.16403721906244756 +fresh-en-2,18,19,Q,strong_half,200,0.10824336409568787,0.13909242302179337,0.1765121206641197,0.1418178764730692,200,0.2890823036432266,0.3539080321788788,0.43973825573921205,0.359026892259717 +fresh-en-2,18,19,Q,random_half,200,0.07490806356072426,0.09903348237276077,0.1346976891160011,0.10324919996783137,200,0.20200481861829758,0.25533226132392883,0.33091104626655576,0.25872911341488364 +fresh-en-2,18,19,Q,all_heads,200,0.11949207335710525,0.1498974710702896,0.18812098503112792,0.15185382388532162,200,0.4275774866342545,0.5230707824230194,0.6239136815071106,0.5230641104280949 +fresh-en-2,18,19,K,weak_half,200,0.020932660065591337,0.030582698993384838,0.04248551279306412,0.031153466934338212,200,0.05957706458866596,0.07829563319683075,0.1082623153924942,0.0792945121973753 +fresh-en-2,18,19,K,strong_half,200,0.054689894616603854,0.07775620371103287,0.1069849781692028,0.07843492612242699,200,0.14461168348789216,0.1971842721104622,0.2742263853549957,0.204391820281744 +fresh-en-2,18,19,K,random_half,200,0.03654505237936974,0.05540618486702442,0.08196513801813125,0.05723076440393925,200,0.09325809925794601,0.1345430165529251,0.19131519496440888,0.13792614135891199 +fresh-en-2,18,19,K,all_heads,200,0.059690576046705246,0.08249938860535622,0.11185120493173598,0.08486094549298287,200,0.2026767432689667,0.26819516718387604,0.38065636754035936,0.2836863350868225 +fresh-en-2,18,19,V,weak_half,200,0.02610942106693983,0.03981200233101845,0.07498983964323998,0.04518444256857038,200,0.0627379596233368,0.09496346861124039,0.18945595324039452,0.10935462744906545 +fresh-en-2,18,19,V,strong_half,200,0.06392522528767586,0.08756846189498901,0.17854304611682892,0.10317282630130649,200,0.13630634546279907,0.19442716240882874,0.46677315831184385,0.2360287469625473 +fresh-en-2,18,19,V,random_half,200,0.03868576288223267,0.06562788039445877,0.13082389682531356,0.07601845710538328,200,0.09023396298289299,0.14420006424188614,0.31895451843738554,0.1711884216219187 +fresh-en-2,18,19,V,all_heads,200,0.07556475624442101,0.10626571998000145,0.20680092275142667,0.1214800363779068,200,0.2008281946182251,0.2949560731649399,0.6489377260208129,0.3453833734989166 +fresh-en-2,18,19,QKV,weak_half,200,0.03747962117195129,0.04750801995396614,0.06492049917578697,0.04899831316433847,200,0.09852159842848777,0.12303999066352844,0.15950531661510464,0.12596030924469231 +fresh-en-2,18,19,QKV,strong_half,200,0.08414000645279884,0.10537290573120117,0.15245690196752548,0.11263453628867864,200,0.21511689573526382,0.2767857611179352,0.38117832541465757,0.2865483629703522 +fresh-en-2,18,19,QKV,random_half,200,0.05771729648113251,0.07969477400183678,0.11096238419413566,0.08262000352144241,200,0.14483172744512557,0.19517602771520615,0.2727850079536438,0.2031930161640048 +fresh-en-2,18,19,QKV,all_heads,200,0.0936530999839306,0.11659436672925949,0.16371700316667556,0.12431363958865405,200,0.31879124641418455,0.3973900228738785,0.5418821156024933,0.4125086717307568 +fresh-en-2,18,19,JS,weak_half,200,0.00022459761530626565,0.0005218450969550759,0.0011758551001548765,0.0006357918498179061,200,0.0005241560225840658,0.001760242914315313,0.009584045130759476,0.003353485557017848 +fresh-en-2,18,19,JS,strong_half,200,0.00600634515285492,0.01611342467367649,0.0316087856888771,0.017971143419854344,200,0.005677597923204303,0.012302326038479805,0.03235330283641815,0.015763604826061053 +fresh-en-2,18,19,JS,random_half,200,0.0016870755120180548,0.00396033632569015,0.012382736708968877,0.006014525844075251,200,0.0015792266465723516,0.005784752545878291,0.023724594525992862,0.010166339651332236 +fresh-en-2,18,19,JS,all_heads,200,0.007024678820744158,0.01698144245892763,0.03315929546952247,0.018972498042276128,200,0.008706798683851958,0.015453939326107502,0.034427409991621964,0.01911709048668854 +fresh-en-2,18,19,sqrtJS,weak_half,200,0.014986581075936556,0.022843893617391586,0.03429039493203163,0.023940483946353196,200,0.058242013305425645,0.091304250061512,0.15120078474283216,0.09784812282770872 +fresh-en-2,18,19,sqrtJS,strong_half,200,0.07749951109290124,0.1269386261701584,0.17778850644826888,0.12777636233717204,200,0.17137150317430497,0.23557989299297333,0.3188486874103546,0.24158662099391223 +fresh-en-2,18,19,sqrtJS,random_half,200,0.041073836386203766,0.06293114274740219,0.1112777329981327,0.07111774884164333,200,0.09542922824621201,0.16221045702695847,0.26258218884468076,0.17152887120842933 +fresh-en-2,18,19,sqrtJS,all_heads,200,0.08380789533257485,0.13031268119812012,0.18209526389837263,0.13180062985047697,200,0.27264648079872134,0.3351541608572006,0.4111552000045776,0.33943474508821964 +fresh-en-2,19,20,Q,weak_half,400,0.022312191128730775,0.04802791401743889,0.09599702060222629,0.05456597681157291,400,0.04187579527497292,0.08784855902194977,0.16117450296878819,0.09743404981680215 +fresh-en-2,19,20,Q,strong_half,400,0.11119192391633988,0.22057317942380905,0.4298665314912796,0.2557695598900318,400,0.14923612773418427,0.2860630452632904,0.5654115796089174,0.33429089684039354 +fresh-en-2,19,20,Q,random_half,400,0.06090257950127125,0.12079918757081032,0.25613137185573587,0.14867963896133005,400,0.08146539926528931,0.17627274245023727,0.37605330646038065,0.21187963229604065 +fresh-en-2,19,20,Q,all_heads,400,0.1252214066684246,0.249314583837986,0.48733526468276994,0.2875955122336745,400,0.19636926800012589,0.3737413287162781,0.7455098628997806,0.43172494566068054 +fresh-en-2,19,20,K,weak_half,400,0.028094514459371566,0.04656004533171654,0.0725024051964283,0.04965831582900137,400,0.05294395983219147,0.08472036570310593,0.13377784341573715,0.09050390279851854 +fresh-en-2,19,20,K,strong_half,400,0.1646551162004471,0.2589903771877289,0.4021450340747833,0.2750159901753068,400,0.205241622030735,0.30381253361701965,0.47861046195030216,0.3279735018312931 +fresh-en-2,19,20,K,random_half,400,0.0829397350549698,0.1418912336230278,0.2693994373083115,0.16358032735064626,400,0.1093521997332573,0.18566123396158218,0.3228598117828369,0.20835675513371826 +fresh-en-2,19,20,K,all_heads,400,0.19314425438642502,0.30003097653388977,0.45124712586402904,0.31452488504350185,400,0.27155905663967134,0.39150239527225494,0.5892058193683625,0.4184774059057236 +fresh-en-2,19,20,V,weak_half,400,0.029132762737572194,0.04903988912701607,0.08434249088168144,0.053273258786648514,400,0.05227456204593182,0.08680631965398788,0.14461768567562103,0.09425703433342278 +fresh-en-2,19,20,V,strong_half,400,0.17621969580650332,0.24413097649812698,0.31640157997608187,0.24336452508345247,400,0.22220313400030137,0.30086277425289154,0.4063667207956314,0.307464641649276 +fresh-en-2,19,20,V,random_half,400,0.08146505728363991,0.14239508658647537,0.22167661190032958,0.1488244271464646,400,0.11248948127031327,0.19711007177829742,0.29035098850727087,0.19938391828909516 +fresh-en-2,19,20,V,all_heads,400,0.18711918741464614,0.277634397149086,0.37151447832584383,0.27765050476416947,400,0.2860807985067368,0.38823360204696655,0.5453923821449281,0.4017216758802533 +fresh-en-2,19,20,QKV,weak_half,400,0.03246900960803032,0.05184360034763813,0.08117592856287958,0.054901327211409806,400,0.058118924126029016,0.09501510858535767,0.14414986222982407,0.09870231025852264 +fresh-en-2,19,20,QKV,strong_half,400,0.1919870838522911,0.2561432123184204,0.35812443494796753,0.27040642458945513,400,0.24674290120601655,0.32401466369628906,0.4613360852003098,0.34376225136220456 +fresh-en-2,19,20,QKV,random_half,400,0.09125136733055114,0.1484479308128357,0.25373237729072573,0.16152641013264657,400,0.12261009588837624,0.20570650696754456,0.33301724791526793,0.21852627370506525 +fresh-en-2,19,20,QKV,all_heads,400,0.21330033242702484,0.294318750500679,0.41035256385803226,0.3051714945212007,400,0.31194029152393343,0.4206542521715164,0.5931314468383789,0.44246456161141395 +fresh-en-2,20,21,Q,weak_half,400,0.010555676277726888,0.01903070230036974,0.051216292381286624,0.024865409838967026,400,0.02870003152638674,0.05306987836956978,0.13216584026813508,0.06736214285949245 +fresh-en-2,20,21,Q,strong_half,400,0.0261221531778574,0.04844912141561508,0.1364738091826439,0.06595774026354775,400,0.06438310965895652,0.11458461731672287,0.32966403663158417,0.15402244267053902 +fresh-en-2,20,21,Q,random_half,400,0.01812243666499853,0.03431669622659683,0.09592265561223035,0.04569481978775002,400,0.04491533450782299,0.08317925781011581,0.22704729437828067,0.11100965557154269 +fresh-en-2,20,21,Q,all_heads,400,0.029353243485093116,0.053196024149656296,0.14993268549442293,0.07410580753814429,400,0.09441880211234092,0.16801995784044266,0.4699925512075425,0.22138458682224155 +fresh-en-2,20,21,K,weak_half,400,0.01840004213154316,0.037367384880781174,0.05413333401083947,0.037309082900173964,400,0.05171629972755909,0.09253136813640594,0.13767281323671346,0.09588593272492289 +fresh-en-2,20,21,K,strong_half,400,0.04176865480840206,0.0944112241268158,0.14389340728521346,0.09589318564161659,400,0.10224279835820198,0.23105346411466599,0.3363164931535721,0.22961508413776754 +fresh-en-2,20,21,K,random_half,400,0.028349611535668374,0.06795335561037064,0.10981716141104698,0.0719620595104061,400,0.07860838323831558,0.15447669476270676,0.24505759179592138,0.1634269771259278 +fresh-en-2,20,21,K,all_heads,400,0.04736917093396187,0.10844092071056366,0.16300380378961565,0.10951345866546035,400,0.15534498691558837,0.3284362852573395,0.46580457091331484,0.3255010176822543 +fresh-en-2,20,21,V,weak_half,400,0.024853584170341492,0.03701223060488701,0.06244433894753458,0.0408287155861035,400,0.06682283729314804,0.09975914284586906,0.14752583354711535,0.10450201814062893 +fresh-en-2,20,21,V,strong_half,400,0.06191869974136353,0.09328923746943474,0.1606333538889885,0.10300714634358883,400,0.17471717298030853,0.23092908412218094,0.3259979486465454,0.24215737663209438 +fresh-en-2,20,21,V,random_half,400,0.04390714466571808,0.06635762751102448,0.11700704842805866,0.07402924767695368,400,0.10987022444605828,0.1623855084180832,0.24752504825592042,0.17247035847976805 +fresh-en-2,20,21,V,all_heads,400,0.06783611699938774,0.10427064076066017,0.18143483549356462,0.11494202020578087,400,0.24610960483551025,0.33345380425453186,0.46363896727561954,0.34665939427912235 +fresh-en-2,20,21,QKV,weak_half,400,0.02272807359695435,0.03423062711954117,0.05155543982982636,0.03634237637743354,400,0.061073803901672365,0.09169922769069672,0.13215160071849824,0.09457933587953449 +fresh-en-2,20,21,QKV,strong_half,400,0.06315441653132439,0.08671626448631287,0.13182060271501542,0.09426669758744538,400,0.153806471824646,0.20942874252796173,0.30736379623413085,0.22347502762451768 +fresh-en-2,20,21,QKV,random_half,400,0.04105055667459965,0.06300100684165955,0.09700513407588006,0.06843057016842068,400,0.10079702585935593,0.14962151646614075,0.2257719486951828,0.1591091011837125 +fresh-en-2,20,21,QKV,all_heads,400,0.07127915620803833,0.09787087142467499,0.14681770503520966,0.10599328986369073,400,0.21920563131570817,0.3048442602157593,0.4313277512788773,0.3180543630942702 +fresh-en-2,21,22,Q,weak_half,400,0.00931399716064334,0.021898302249610424,0.038178750127553944,0.022938319473760203,400,0.024124759621918204,0.060972779989242554,0.09920559152960778,0.061369043102022264 +fresh-en-2,21,22,Q,strong_half,400,0.016500892490148546,0.0402578953653574,0.08211425840854647,0.045730669943150136,400,0.04745509326457977,0.11513236537575722,0.21439537703990938,0.12290293152444065 +fresh-en-2,21,22,Q,random_half,400,0.013270015548914672,0.02910253405570984,0.0632727697491646,0.034245808745035904,400,0.03613910302519799,0.08638086169958115,0.15586436539888385,0.09263376434799284 +fresh-en-2,21,22,Q,all_heads,400,0.02034194059669972,0.04925324581563473,0.09388896599411967,0.053451861711218955,400,0.07465631887316704,0.1763467639684677,0.32160365581512457,0.18427197535522283 +fresh-en-2,21,22,K,weak_half,400,0.01167916776612401,0.031119454652071,0.047005798295140275,0.030800221665995196,400,0.03175648916512728,0.0807696133852005,0.11805720552802086,0.07796416678000241 +fresh-en-2,21,22,K,strong_half,400,0.019522262178361418,0.059725016355514526,0.09181661084294321,0.0574098537536338,400,0.055424225702881816,0.15754570066928864,0.2366847291588784,0.15217328746803105 +fresh-en-2,21,22,K,random_half,400,0.016236221604049205,0.0486386027187109,0.0764220833778382,0.0477249886165373,400,0.04252871125936508,0.11888515576720238,0.17920466959476472,0.11570134827401489 +fresh-en-2,21,22,K,all_heads,400,0.022847104445099832,0.06825323775410652,0.10247391909360885,0.06556730627082288,400,0.08607032299041747,0.2373434379696846,0.34947484731674194,0.23013745415955783 +fresh-en-2,21,22,V,weak_half,400,0.018746725097298623,0.027998900040984154,0.045372044295072564,0.030778603358194233,400,0.05090965293347836,0.07261796668171883,0.11219931393861772,0.07735752198845149 +fresh-en-2,21,22,V,strong_half,400,0.03639355227351189,0.05382493883371353,0.0863565929234028,0.05816497864201665,400,0.09905741214752198,0.14451340585947037,0.2221935823559761,0.1546433382295072 +fresh-en-2,21,22,V,random_half,400,0.02803809717297554,0.04339424893260002,0.07448351308703424,0.04809886131668464,400,0.07428466007113457,0.11115363612771034,0.16771485507488257,0.11734896048903465 +fresh-en-2,21,22,V,all_heads,400,0.042593063414096834,0.06466145813465118,0.10053480416536331,0.0677937503810972,400,0.15097150653600694,0.21877125650644302,0.3306225121021271,0.23200086012482643 +fresh-en-2,21,22,QKV,weak_half,400,0.017043645307421685,0.028997579589486122,0.04191418215632439,0.029337040984537453,400,0.04864052049815656,0.07408550381660461,0.10512230396270754,0.0756365757714957 +fresh-en-2,21,22,QKV,strong_half,400,0.03159454166889191,0.055058933794498444,0.08377886638045311,0.056292985319159924,400,0.08917102292180062,0.14804424345493317,0.21686223447322847,0.1510042577330023 +fresh-en-2,21,22,QKV,random_half,400,0.0246064318343997,0.043845683336257935,0.06895383670926095,0.04548375173471868,400,0.06613964959979057,0.11111726611852646,0.16385638415813447,0.11412865965627134 +fresh-en-2,21,22,QKV,all_heads,400,0.037201641499996184,0.06342905387282372,0.09434762373566628,0.06490597075782717,400,0.1377473443746567,0.227631613612175,0.3231204360723496,0.22664083315059544 +fresh-en-2,22,23,Q,weak_half,200,0.0389533881098032,0.04999127797782421,0.06624509766697884,0.05176370607689023,200,0.10141095444560051,0.13198532909154892,0.17392164170742036,0.13561811145395042 +fresh-en-2,22,23,Q,strong_half,200,0.12220989093184471,0.16683386266231537,0.21567520648241043,0.16803053427487613,200,0.30210573971271515,0.3655426949262619,0.4671970248222351,0.38033226907253265 +fresh-en-2,22,23,Q,random_half,200,0.0664086565375328,0.1100059412419796,0.167375610768795,0.11563154835253954,200,0.16592652052640916,0.2501985728740692,0.34564377963542936,0.25787122543901203 +fresh-en-2,22,23,Q,all_heads,200,0.1299076646566391,0.17173487693071365,0.22716087400913237,0.1764223525673151,200,0.40774807929992674,0.5050136148929596,0.6341044545173644,0.5159503798186779 +fresh-en-2,22,23,K,weak_half,200,0.028949798457324507,0.041822660714387894,0.051199693232774734,0.04099917070940137,200,0.07369395121932029,0.10131559520959854,0.1330968290567398,0.103946040160954 +fresh-en-2,22,23,K,strong_half,200,0.10683702677488327,0.15698014199733734,0.2438298910856247,0.16466735070571303,200,0.29068297445774083,0.3548680543899536,0.47747019231319426,0.3690085490047932 +fresh-en-2,22,23,K,random_half,200,0.05264297462999821,0.09579751640558243,0.17656457275152204,0.10681137308478356,200,0.1324266955256462,0.22148170322179794,0.34936087727546694,0.2310579714924097 +fresh-en-2,22,23,K,all_heads,200,0.11361701637506486,0.15924646705389023,0.24686982482671735,0.16891771242022513,200,0.37558896243572243,0.47076329588890076,0.5818997383117676,0.4729545897245407 +fresh-en-2,22,23,V,weak_half,200,0.016384299844503403,0.021196365356445312,0.03280744329094887,0.022833937238901855,200,0.04060429334640503,0.05540424399077892,0.07223494350910187,0.05574538918212056 +fresh-en-2,22,23,V,strong_half,200,0.058689451590180394,0.1103997752070427,0.24136817008256886,0.13169469757005572,200,0.19117704033851624,0.26299384236335754,0.430972039699554,0.28254489801824095 +fresh-en-2,22,23,V,random_half,200,0.03152900338172913,0.05801411159336567,0.16173454821109773,0.080125452009961,200,0.08816137313842774,0.1406097188591957,0.27806933522224425,0.16346936713904142 +fresh-en-2,22,23,V,all_heads,200,0.060079242661595346,0.10810219869017601,0.23140850961208315,0.12932032376527786,200,0.23842049539089202,0.3194464296102524,0.5010916978120801,0.33829028517007825 +fresh-en-2,22,23,QKV,weak_half,200,0.031150876171886922,0.039635736495256424,0.05141470767557621,0.04053872333839536,200,0.07782209068536758,0.10610911250114441,0.13214430958032608,0.10524143025279045 +fresh-en-2,22,23,QKV,strong_half,200,0.11047962605953217,0.14979004859924316,0.23912768363952636,0.1602185521647334,200,0.2874346524477005,0.35592761635780334,0.4805771172046661,0.36429339230060576 +fresh-en-2,22,23,QKV,random_half,200,0.05558180138468742,0.09700236469507217,0.16592046171426772,0.10514739274978638,200,0.1389492630958557,0.22309546917676926,0.32501681745052335,0.2304717279225588 +fresh-en-2,22,23,QKV,all_heads,200,0.11478826925158502,0.15673191845417023,0.23817770928144455,0.16404451347887516,200,0.3650020569562912,0.46348312497138977,0.5915480077266693,0.4695348219573498 +fresh-en-2,22,23,JS,weak_half,200,2.6631626860762486e-05,0.00034265876456629485,0.0006805745768360792,0.00035941591977916687,200,3.4746906021609905e-05,0.0004492008447414264,0.0010524250916205346,0.0005324402541327799 +fresh-en-2,22,23,JS,strong_half,200,0.0007596941781230271,0.006165757542476058,0.015315448399633166,0.007219829299901903,200,0.0011620680801570416,0.006236781133338809,0.012310632690787315,0.006744058378098998 +fresh-en-2,22,23,JS,random_half,200,0.0005207840586081147,0.005767028545960784,0.012176248803734779,0.0059941811004591726,200,0.0005131326441187412,0.002295633195899427,0.007598261302337049,0.0034575933038649962 +fresh-en-2,22,23,JS,all_heads,200,0.0010184155078604817,0.006103033199906349,0.014358862116932864,0.007104080252975109,200,0.0012135659577324988,0.0072148703038692474,0.012908982299268245,0.007276498668798013 +fresh-en-2,22,23,sqrtJS,weak_half,200,0.005160006461665035,0.01851104386150837,0.026087502948939797,0.017333876283373684,200,0.014976695086807013,0.053745828568935394,0.0754989743232727,0.04983359294943512 +fresh-en-2,22,23,sqrtJS,strong_half,200,0.027562499418854716,0.07852231338620186,0.12375382557511329,0.0785425080661662,200,0.07881802022457124,0.18025030195713043,0.23707639724016188,0.17070898147299884 +fresh-en-2,22,23,sqrtJS,random_half,200,0.0228204183280468,0.0759400837123394,0.11034593060612678,0.07057823308045044,200,0.045493650808930405,0.10586847364902496,0.17022707164287568,0.10990765505470336 +fresh-en-2,22,23,sqrtJS,all_heads,200,0.03191238604485989,0.07812189310789108,0.11982487365603445,0.07817319827154279,200,0.0907830648124218,0.23895534127950668,0.30426531136035917,0.2205425751209259 diff --git a/results/head-dominance/mainline-activation-tests.txt b/results/head-dominance/mainline-activation-tests.txt new file mode 100644 index 0000000..1c3e190 --- /dev/null +++ b/results/head-dominance/mainline-activation-tests.txt @@ -0,0 +1,7 @@ +test_known_activity (tests.test_activation.ActivationTests.test_known_activity) ... ok +test_zero_and_scale (tests.test_activation.ActivationTests.test_zero_and_scale) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 0.038s + +OK diff --git a/results/head-dominance/observations.npz b/results/head-dominance/observations.npz new file mode 100644 index 0000000..0fbb74d Binary files /dev/null and b/results/head-dominance/observations.npz differ diff --git a/results/head-dominance/primary-analysis-summary.json b/results/head-dominance/primary-analysis-summary.json new file mode 100644 index 0000000..a37aaf3 --- /dev/null +++ b/results/head-dominance/primary-analysis-summary.json @@ -0,0 +1,1294 @@ +{ + "by_recipient_type": { + "Q": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.42808762192726135, + "median": 0.8017966151237488, + "p90": 1.2796676754951481, + "mean": 0.828383005860572 + }, + "max_absolute": { + "count": 4800, + "p10": 0.07022316381335258, + "median": 0.13597501814365387, + "p90": 0.29393495619297033, + "mean": 0.1614389501255937 + }, + "effective_heads": { + "count": 4800, + "p10": 7.634387731552124, + "median": 10.829116821289062, + "p90": 12.903363132476807, + "mean": 10.545973850339651 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.47714923322200775, + "median": 0.6768198013305664, + "p90": 0.8064601957798004, + "mean": 0.6591233656462282 + }, + "concentration": { + "count": 4800, + "p10": 0.015999119356274606, + "median": 0.031833209097385406, + "p90": 0.07305203005671501, + "mean": 0.038962758334334165 + }, + "top1": { + "count": 4800, + "p10": 0.13040318638086318, + "median": 0.1768590658903122, + "p90": 0.27592835426330575, + "mean": 0.1917892166459933 + }, + "top25": { + "count": 4800, + "p10": 0.4181676685810089, + "median": 0.49224844574928284, + "p90": 0.5894871711730958, + "mean": 0.4984907660695414 + }, + "top50": { + "count": 4800, + "p10": 0.6820279717445373, + "median": 0.7461760938167572, + "p90": 0.8289147377014161, + "mean": 0.7511068653936187 + }, + "bottom25": { + "count": 4800, + "p10": 0.04171919822692871, + "median": 0.0870458073914051, + "p90": 0.12668813914060592, + "mean": 0.0858515575454415 + }, + "heads90": { + "count": 4800, + "p10": 10.0, + "median": 12.0, + "p90": 13.0, + "mean": 11.924375 + } + }, + "linear_attention": { + "total": { + "count": 27200, + "p10": 0.1771158680319786, + "median": 0.7782308757305145, + "p90": 1.8638553977012642, + "mean": 0.9841912085086326 + }, + "max_absolute": { + "count": 27200, + "p10": 0.027340889908373356, + "median": 0.13787434250116348, + "p90": 0.4472817718982699, + "mean": 0.2126326917772873 + }, + "effective_heads": { + "count": 27200, + "p10": 5.968966674804688, + "median": 10.08900499343872, + "p90": 13.287356853485107, + "mean": 9.637892792207353 + }, + "effective_fraction": { + "count": 27200, + "p10": 0.5126844048500061, + "median": 0.7466321587562561, + "p90": 0.8671843826770783, + "mean": 0.7170763921009048 + }, + "concentration": { + "count": 27200, + "p10": 0.012945951707661153, + "median": 0.02890323381870985, + "p90": 0.07301022931933407, + "mean": 0.03798074354623489 + }, + "top1": { + "count": 27200, + "p10": 0.12049156576395034, + "median": 0.18831469118595123, + "p90": 0.3018690258264542, + "mean": 0.20151331216672105 + }, + "top25": { + "count": 27200, + "p10": 0.38046165108680724, + "median": 0.453884556889534, + "p90": 0.5794604063034058, + "mean": 0.4684076986615272 + }, + "top50": { + "count": 27200, + "p10": 0.650843471288681, + "median": 0.7193152010440826, + "p90": 0.8102161049842835, + "mean": 0.7255355768234414 + }, + "bottom25": { + "count": 27200, + "p10": 0.05939290076494217, + "median": 0.10252666845917702, + "p90": 0.14262291491031648, + "mean": 0.1017356713594499 + }, + "heads90": { + "count": 27200, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.579007352941176 + } + } + }, + "K": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.2932938843965531, + "median": 0.751684695482254, + "p90": 1.2453256130218517, + "mean": 0.7491105496883392 + }, + "max_absolute": { + "count": 4800, + "p10": 0.05981316082179546, + "median": 0.16251461952924728, + "p90": 0.29028533995151523, + "mean": 0.17104425797394165 + }, + "effective_heads": { + "count": 4800, + "p10": 6.421846866607666, + "median": 8.99651575088501, + "p90": 11.717314338684085, + "mean": 9.006867933471998 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.4013654291629791, + "median": 0.5622822344303131, + "p90": 0.7323321461677553, + "mean": 0.5629292458419999 + }, + "concentration": { + "count": 4800, + "p10": 0.024366708286106587, + "median": 0.05189775675535202, + "p90": 0.09943302348256117, + "mean": 0.05819399481251215 + }, + "top1": { + "count": 4800, + "p10": 0.15217750519514084, + "median": 0.22362081706523895, + "p90": 0.333113169670105, + "mean": 0.2327083595407506 + }, + "top25": { + "count": 4800, + "p10": 0.46408111453056333, + "median": 0.554540365934372, + "p90": 0.6376730680465699, + "mean": 0.5528365189085404 + }, + "top50": { + "count": 4800, + "p10": 0.7256738007068634, + "median": 0.7895856201648712, + "p90": 0.8492424726486212, + "mean": 0.7887261419494946 + }, + "bottom25": { + "count": 4800, + "p10": 0.037170265614986424, + "median": 0.06792030483484268, + "p90": 0.10355381369590763, + "mean": 0.06925618241191842 + }, + "heads90": { + "count": 4800, + "p10": 10.0, + "median": 11.0, + "p90": 13.0, + "mean": 11.344166666666666 + } + }, + "linear_attention": { + "total": { + "count": 27200, + "p10": 0.27437427937984465, + "median": 0.7400951087474823, + "p90": 1.6867972612380997, + "mean": 0.9442492292869343 + }, + "max_absolute": { + "count": 27200, + "p10": 0.04476405493915081, + "median": 0.12380896881222725, + "p90": 0.3700851738452914, + "mean": 0.18057506222894196 + }, + "effective_heads": { + "count": 27200, + "p10": 6.289550733566284, + "median": 10.748710632324219, + "p90": 13.59198980331421, + "mean": 10.086356610287638 + }, + "effective_fraction": { + "count": 27200, + "p10": 0.567463606595993, + "median": 0.7731030285358429, + "p90": 0.8852756083011627, + "mean": 0.7487537822614917 + }, + "concentration": { + "count": 27200, + "p10": 0.01110830344259739, + "median": 0.024465462192893028, + "p90": 0.059676705300807986, + "mean": 0.03137563373759108 + }, + "top1": { + "count": 27200, + "p10": 0.11619977951049805, + "median": 0.17642024159431458, + "p90": 0.2702268481254578, + "mean": 0.18697812774930805 + }, + "top25": { + "count": 27200, + "p10": 0.3685951292514801, + "median": 0.4383372962474823, + "p90": 0.5479569613933564, + "mean": 0.44980560849694645 + }, + "top50": { + "count": 27200, + "p10": 0.6412311494350433, + "median": 0.7067530751228333, + "p90": 0.7897655248641968, + "mean": 0.7119768214488731 + }, + "bottom25": { + "count": 27200, + "p10": 0.06756735369563102, + "median": 0.10880804806947708, + "p90": 0.14901476353406906, + "mean": 0.10839668871242526 + }, + "heads90": { + "count": 27200, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.763308823529412 + } + } + }, + "V": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.2459064647555352, + "median": 0.7926892042160034, + "p90": 1.4473202824592595, + "mean": 0.8259200246507923 + }, + "max_absolute": { + "count": 4800, + "p10": 0.04807887561619282, + "median": 0.14048674702644348, + "p90": 0.3213535517454149, + "mean": 0.1687076044936354 + }, + "effective_heads": { + "count": 4800, + "p10": 6.27513017654419, + "median": 10.059541702270508, + "p90": 13.006668281555179, + "mean": 9.797848190466564 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.39219563603401186, + "median": 0.6287213563919067, + "p90": 0.8129167675971987, + "mean": 0.6123655119041602 + }, + "concentration": { + "count": 4800, + "p10": 0.01534255063161254, + "median": 0.039368655532598495, + "p90": 0.1033165372908116, + "mean": 0.054241599862386164 + }, + "top1": { + "count": 4800, + "p10": 0.12827315628528596, + "median": 0.1957768052816391, + "p90": 0.32790936529636394, + "mean": 0.21723997221017877 + }, + "top25": { + "count": 4800, + "p10": 0.40897682309150696, + "median": 0.5191850960254669, + "p90": 0.6538566529750826, + "mean": 0.5266173542042574 + }, + "top50": { + "count": 4800, + "p10": 0.6763901829719543, + "median": 0.7685472369194031, + "p90": 0.8522575080394748, + "mean": 0.7653420636554559 + }, + "bottom25": { + "count": 4800, + "p10": 0.038462628796696666, + "median": 0.07527097687125206, + "p90": 0.1310943305492402, + "mean": 0.08093525390451153 + }, + "heads90": { + "count": 4800, + "p10": 10.0, + "median": 12.0, + "p90": 13.0, + "mean": 11.665 + } + }, + "linear_attention": { + "total": { + "count": 27200, + "p10": 0.29220492839813234, + "median": 0.5876882374286652, + "p90": 1.7076982617378238, + "mean": 0.8579155112553717 + }, + "max_absolute": { + "count": 27200, + "p10": 0.04601397477090359, + "median": 0.11557785794138908, + "p90": 0.3890607744455338, + "mean": 0.18123242851627497 + }, + "effective_heads": { + "count": 27200, + "p10": 5.704750633239746, + "median": 9.907225608825684, + "p90": 13.273716259002686, + "mean": 9.52269080659046 + }, + "effective_fraction": { + "count": 27200, + "p10": 0.5001892507076263, + "median": 0.736346960067749, + "p90": 0.8675740957260132, + "mean": 0.7092839469459347 + }, + "concentration": { + "count": 27200, + "p10": 0.012901593837887049, + "median": 0.029933489859104156, + "p90": 0.07959845960140231, + "mean": 0.03960282899909276 + }, + "top1": { + "count": 27200, + "p10": 0.12066357731819152, + "median": 0.1884898841381073, + "p90": 0.30654583871364594, + "mean": 0.20284635099856294 + }, + "top25": { + "count": 27200, + "p10": 0.38022677004337313, + "median": 0.4595302492380142, + "p90": 0.5857594251632692, + "mean": 0.47216263436109707 + }, + "top50": { + "count": 27200, + "p10": 0.6528049886226654, + "median": 0.7263502180576324, + "p90": 0.8170590281486512, + "mean": 0.7314765423689695 + }, + "bottom25": { + "count": 27200, + "p10": 0.055867613106966016, + "median": 0.09783932939171791, + "p90": 0.14103657454252244, + "mean": 0.09802515388039276 + }, + "heads90": { + "count": 27200, + "p10": 6.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.472205882352942 + } + } + }, + "QKV": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.3599545568227768, + "median": 0.8310977220535278, + "p90": 1.3057362198829652, + "mean": 0.8320240748580545 + }, + "max_absolute": { + "count": 4800, + "p10": 0.06178740113973618, + "median": 0.1471318155527115, + "p90": 0.2991165637969971, + "mean": 0.16522609861179566 + }, + "effective_heads": { + "count": 4800, + "p10": 7.475884056091308, + "median": 10.319912433624268, + "p90": 12.712833499908447, + "mean": 10.20285238613685 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.46724275350570676, + "median": 0.6449945271015167, + "p90": 0.7945520937442779, + "mean": 0.6376782741335532 + }, + "concentration": { + "count": 4800, + "p10": 0.017238049022853374, + "median": 0.03669338673353195, + "p90": 0.07601434364914894, + "mean": 0.04257073526978881 + }, + "top1": { + "count": 4800, + "p10": 0.13321341276168824, + "median": 0.18747321516275406, + "p90": 0.28328865170478823, + "mean": 0.1998321647165964 + }, + "top25": { + "count": 4800, + "p10": 0.4247431606054306, + "median": 0.5103605389595032, + "p90": 0.5986294209957124, + "mean": 0.5111861900550624 + }, + "top50": { + "count": 4800, + "p10": 0.6898793756961823, + "median": 0.7608179152011871, + "p90": 0.826508241891861, + "mean": 0.759507774884502 + }, + "bottom25": { + "count": 4800, + "p10": 0.043933799117803575, + "median": 0.0804036296904087, + "p90": 0.12289210483431817, + "mean": 0.08222739746629183 + }, + "heads90": { + "count": 4800, + "p10": 10.0, + "median": 12.0, + "p90": 13.0, + "mean": 11.84 + } + }, + "linear_attention": { + "total": { + "count": 27200, + "p10": 0.300345253944397, + "median": 0.780917763710022, + "p90": 1.794381380081177, + "mean": 0.9848778233173139 + }, + "max_absolute": { + "count": 27200, + "p10": 0.04775628037750721, + "median": 0.12801389396190643, + "p90": 0.40935705900192265, + "mean": 0.19344240318685907 + }, + "effective_heads": { + "count": 27200, + "p10": 6.223408126831055, + "median": 10.616299629211426, + "p90": 13.700624370574952, + "mean": 10.054770943066654 + }, + "effective_fraction": { + "count": 27200, + "p10": 0.551340651512146, + "median": 0.7776066660881042, + "p90": 0.8834639668464661, + "mean": 0.7460366055385812 + }, + "concentration": { + "count": 27200, + "p10": 0.010811386536806822, + "median": 0.02463168278336525, + "p90": 0.0634766072034836, + "mean": 0.032149144387093494 + }, + "top1": { + "count": 27200, + "p10": 0.11426723748445511, + "median": 0.17895939201116562, + "p90": 0.2811527192592621, + "mean": 0.18969931307632257 + }, + "top25": { + "count": 27200, + "p10": 0.3703198492527008, + "median": 0.43639612197875977, + "p90": 0.5559773981571198, + "mean": 0.45095220466199165 + }, + "top50": { + "count": 27200, + "p10": 0.6413009822368622, + "median": 0.7052080035209656, + "p90": 0.7924475073814392, + "mean": 0.7119353248003651 + }, + "bottom25": { + "count": 27200, + "p10": 0.06760622188448906, + "median": 0.11000294238328934, + "p90": 0.1492564007639885, + "mean": 0.10894173061254311 + }, + "heads90": { + "count": 27200, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.776801470588236 + } + } + }, + "JS": { + "full_attention": { + "total": { + "count": 4800, + "p10": 0.006019792705774307, + "median": 0.02659580297768116, + "p90": 0.07109484225511555, + "mean": 0.03422935250632387 + }, + "max_absolute": { + "count": 4800, + "p10": 0.0024493230041116474, + "median": 0.009726522956043482, + "p90": 0.03804903179407121, + "mean": 0.01699844059891954 + }, + "effective_heads": { + "count": 4800, + "p10": 1.8775771021842957, + "median": 3.901782989501953, + "p90": 6.948997354507451, + "mean": 4.195899176249902 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.11734856888651848, + "median": 0.24386143684387207, + "p90": 0.4343123346567157, + "mean": 0.26224369851561885 + }, + "concentration": { + "count": 4800, + "p10": 0.08683274388313295, + "median": 0.20671262592077255, + "p90": 0.5014413654804231, + "mean": 0.25655510467089093 + }, + "top1": { + "count": 4800, + "p10": 0.24280447959899903, + "median": 0.4230606108903885, + "p90": 0.7145513713359836, + "mean": 0.45466951202290756 + }, + "top25": { + "count": 4800, + "p10": 0.6646065950393677, + "median": 0.8229818344116211, + "p90": 0.9326550900936127, + "mean": 0.8089195707378288 + }, + "top50": { + "count": 4800, + "p10": 0.8803883492946625, + "median": 0.9476927518844604, + "p90": 0.9820005834102631, + "mean": 0.9382064804434777 + }, + "bottom25": { + "count": 4800, + "p10": 0.003459757659584284, + "median": 0.011073274537920952, + "p90": 0.03202965445816518, + "mean": 0.014963302712030781 + }, + "heads90": { + "count": 4800, + "p10": 3.0, + "median": 6.0, + "p90": 9.0, + "mean": 6.310208333333334 + } + } + } + }, + "full_attention_source_indices": [ + 2, + 6, + 10, + 14, + 18, + 22 + ], + "paired_joint": { + "Q": { + "weak_less_than_strong_fraction": 0.9558125, + "weak_less_than_random_fraction": 0.92478125, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 32000, + "p10": 1.2414084553718567, + "median": 2.406548857688904, + "p90": 4.8988345146179215, + "mean": 2.8476133695249444 + }, + "random_over_weak": { + "count": 32000, + "p10": 1.0663463115692138, + "median": 1.7550507187843323, + "p90": 3.4048652887344364, + "mean": 2.0716914434591307 + } + }, + "K": { + "weak_less_than_strong_fraction": 0.94053125, + "weak_less_than_random_fraction": 0.93659375, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 32000, + "p10": 1.1599948525428772, + "median": 2.2635676860809326, + "p90": 4.8068671226501465, + "mean": 2.721669250183739 + }, + "random_over_weak": { + "count": 32000, + "p10": 1.0976211547851562, + "median": 1.7731841802597046, + "p90": 3.4223729372024536, + "mean": 2.0756989874048157 + } + }, + "V": { + "weak_less_than_strong_fraction": 0.9139375, + "weak_less_than_random_fraction": 0.89215625, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 32000, + "p10": 1.0545674324035645, + "median": 2.339610457420349, + "p90": 5.230419206619263, + "mean": 2.8710654100831596 + }, + "random_over_weak": { + "count": 32000, + "p10": 0.9815070390701294, + "median": 1.7466103434562683, + "p90": 3.7226275205612196, + "mean": 2.160556613523513 + } + }, + "QKV": { + "weak_less_than_strong_fraction": 0.95575, + "weak_less_than_random_fraction": 0.9521875, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 32000, + "p10": 1.2344148278236389, + "median": 2.357925534248352, + "p90": 4.5909470558166525, + "mean": 2.7040380254182965 + }, + "random_over_weak": { + "count": 32000, + "p10": 1.1472956657409668, + "median": 1.7747376561164856, + "p90": 3.2936715126037615, + "mean": 2.046528765618801 + } + }, + "JS": { + "weak_less_than_strong_fraction": 0.9970833333333333, + "weak_less_than_random_fraction": 0.988125, + "zero_weak_effect_count": 0, + "strong_over_weak": { + "count": 4800, + "p10": 4.6294151782989506, + "median": 18.771913528442383, + "p90": 68.13132858276367, + "mean": 32.662432227910806 + }, + "random_over_weak": { + "count": 4800, + "p10": 2.710283899307251, + "median": 10.478245258331299, + "p90": 39.4547286987305, + "mean": 19.306766499467194 + } + } + }, + "pooling": "query7..31, per text/token/recipient-query-head; descriptive dependent observations, not independent samples", + "concentration": { + "Q": { + "total": { + "count": 32000, + "p10": 0.19806515499949456, + "median": 0.7841357989236712, + "p90": 1.727399396291003, + "mean": 0.9608199778404878 + }, + "max_absolute": { + "count": 32000, + "p10": 0.03213169947266579, + "median": 0.13753367960453033, + "p90": 0.40950461924076104, + "mean": 0.20495363052953325 + }, + "effective_heads": { + "count": 32000, + "p10": 6.074677358285013, + "median": 10.278813612228042, + "p90": 13.231621453644234, + "mean": 9.774104951134538 + }, + "effective_fraction": { + "count": 32000, + "p10": 0.5059540464573292, + "median": 0.736010784821391, + "p90": 0.8612895367324331, + "mean": 0.7083834381455979 + }, + "concentration": { + "count": 32000, + "p10": 0.01331159273601288, + "median": 0.029374472335226654, + "p90": 0.07302650438478069, + "mean": 0.03812804576861353 + }, + "top1": { + "count": 32000, + "p10": 0.12190424615545024, + "median": 0.1862308612148443, + "p90": 0.2983967835528167, + "mean": 0.20005469782669785 + }, + "top25": { + "count": 32000, + "p10": 0.38400657936407756, + "median": 0.45974826740579305, + "p90": 0.5813789312182575, + "mean": 0.4729201587141697 + }, + "top50": { + "count": 32000, + "p10": 0.6545684499580247, + "median": 0.7233334799076764, + "p90": 0.8135213374924113, + "mean": 0.7293712701244147 + }, + "bottom25": { + "count": 32000, + "p10": 0.056496162230507345, + "median": 0.10028684220891418, + "p90": 0.14090863982385138, + "mean": 0.09935305428362357 + }, + "heads90": { + "count": 32000, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.7808125 + } + }, + "K": { + "total": { + "count": 32000, + "p10": 0.27868743662256745, + "median": 0.7418987276032567, + "p90": 1.5808700304012746, + "mean": 0.9149784272735005 + }, + "max_absolute": { + "count": 32000, + "p10": 0.04672095738351345, + "median": 0.12887775897979736, + "p90": 0.34267061948776245, + "mean": 0.17914544159069193 + }, + "effective_heads": { + "count": 32000, + "p10": 6.2997924969825645, + "median": 10.282293972258437, + "p90": 13.459263973201868, + "mean": 9.924433305753986 + }, + "effective_fraction": { + "count": 32000, + "p10": 0.5169652544140182, + "median": 0.7494478491534898, + "p90": 0.8792124936928435, + "mean": 0.7208801016057295 + }, + "concentration": { + "count": 32000, + "p10": 0.011776948469577102, + "median": 0.027261487005195033, + "p90": 0.0685273479698932, + "mean": 0.03539838788532774 + }, + "top1": { + "count": 32000, + "p10": 0.11868254832556942, + "median": 0.1837866249324059, + "p90": 0.2811234081621101, + "mean": 0.19383766248805132 + }, + "top25": { + "count": 32000, + "p10": 0.37291786446716907, + "median": 0.4519532440455542, + "p90": 0.5801074854843632, + "mean": 0.4652602449275946 + }, + "top50": { + "count": 32000, + "p10": 0.6453049612946737, + "median": 0.7175863432426386, + "p90": 0.8124186772934331, + "mean": 0.7234892194956966 + }, + "bottom25": { + "count": 32000, + "p10": 0.05735948077175399, + "median": 0.1032195714581893, + "p90": 0.14622727669722033, + "mean": 0.10252561275154358 + }, + "heads90": { + "count": 32000, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.8504375 + } + }, + "V": { + "total": { + "count": 32000, + "p10": 0.2812398451380432, + "median": 0.6038189655519091, + "p90": 1.645845576003194, + "mean": 0.8531161886132836 + }, + "max_absolute": { + "count": 32000, + "p10": 0.04643451571464539, + "median": 0.1182936355471611, + "p90": 0.3698546379804613, + "mean": 0.17935370491287903 + }, + "effective_heads": { + "count": 32000, + "p10": 5.736491743713151, + "median": 9.942905498299558, + "p90": 13.239749202567198, + "mean": 9.563964412704575 + }, + "effective_fraction": { + "count": 32000, + "p10": 0.47907725223804926, + "median": 0.7227435911028347, + "p90": 0.8621985150352091, + "mean": 0.6947461816057294 + }, + "concentration": { + "count": 32000, + "p10": 0.013188755715480607, + "median": 0.03110799980364811, + "p90": 0.08225759989950268, + "mean": 0.0417986446199604 + }, + "top1": { + "count": 32000, + "p10": 0.12161388449889787, + "median": 0.1896916578538292, + "p90": 0.30860335061787136, + "mean": 0.20500539421837363 + }, + "top25": { + "count": 32000, + "p10": 0.3836189886588612, + "median": 0.4664460271280716, + "p90": 0.5998502376306984, + "mean": 0.4803308423023778 + }, + "top50": { + "count": 32000, + "p10": 0.6551911500035564, + "median": 0.7314303909565261, + "p90": 0.8252102923927693, + "mean": 0.736556370556187 + }, + "bottom25": { + "count": 32000, + "p10": 0.05101045030368696, + "median": 0.09549013543061644, + "p90": 0.13985548819761448, + "mean": 0.09546166889828221 + }, + "heads90": { + "count": 32000, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.651125 + } + }, + "QKV": { + "total": { + "count": 32000, + "p10": 0.31270650094375013, + "median": 0.7898262748494744, + "p90": 1.689432829990983, + "mean": 0.96194976092564 + }, + "max_absolute": { + "count": 32000, + "p10": 0.0506718147546053, + "median": 0.13108564168214798, + "p90": 0.375281447172165, + "mean": 0.18920995750059955 + }, + "effective_heads": { + "count": 32000, + "p10": 6.304201972468572, + "median": 10.534389403054181, + "p90": 13.605718976137743, + "mean": 10.076983159324206 + }, + "effective_fraction": { + "count": 32000, + "p10": 0.531184131354781, + "median": 0.7600632811713559, + "p90": 0.8777116494832276, + "mean": 0.7297828558447914 + }, + "concentration": { + "count": 32000, + "p10": 0.01127297981394185, + "median": 0.026350765843306763, + "p90": 0.06576838727607895, + "mean": 0.03371238303288477 + }, + "top1": { + "count": 32000, + "p10": 0.11599946413523492, + "median": 0.1805094629907935, + "p90": 0.2813719110798307, + "mean": 0.19121924080514013 + }, + "top25": { + "count": 32000, + "p10": 0.37423556131777447, + "median": 0.446468832512783, + "p90": 0.5689850647038045, + "mean": 0.4599873026303993 + }, + "top50": { + "count": 32000, + "p10": 0.6450496968093983, + "median": 0.71281259036529, + "p90": 0.8019362921144723, + "mean": 0.7190711923720653 + }, + "bottom25": { + "count": 32000, + "p10": 0.06218593053663734, + "median": 0.10599167602648567, + "p90": 0.14689275149483338, + "mean": 0.10493458066537943 + }, + "heads90": { + "count": 32000, + "p10": 7.0, + "median": 12.0, + "p90": 13.0, + "mean": 10.93628125 + } + }, + "JS": { + "total": { + "count": 4800, + "p10": 0.006019792613687969, + "median": 0.026595803305099253, + "p90": 0.07109484240791066, + "mean": 0.03422935252324174 + }, + "max_absolute": { + "count": 4800, + "p10": 0.0024493230041116474, + "median": 0.009726522956043482, + "p90": 0.03804903179407121, + "mean": 0.01699844059891954 + }, + "effective_heads": { + "count": 4800, + "p10": 1.8775771328772404, + "median": 3.901782998659897, + "p90": 6.9489972185767925, + "mean": 4.1958991748153185 + }, + "effective_fraction": { + "count": 4800, + "p10": 0.11734857080482752, + "median": 0.24386143741624355, + "p90": 0.43431232616104953, + "mean": 0.2622436984259574 + }, + "concentration": { + "count": 4800, + "p10": 0.08683274490451195, + "median": 0.20671262373235755, + "p90": 0.5014413860763198, + "mean": 0.25655510469396303 + }, + "top1": { + "count": 4800, + "p10": 0.24280447987972617, + "median": 0.4230606060703387, + "p90": 0.7145513508216859, + "mean": 0.45466951196884586 + }, + "top25": { + "count": 4800, + "p10": 0.6646066107988152, + "median": 0.8229818456850793, + "p90": 0.9326550608746287, + "mean": 0.808919570864033 + }, + "top50": { + "count": 4800, + "p10": 0.8803883543181658, + "median": 0.9476927560262463, + "p90": 0.9820005650641489, + "mean": 0.9382064806256976 + }, + "bottom25": { + "count": 4800, + "p10": 0.0034597576965423498, + "median": 0.01107327468889098, + "p90": 0.03202965476590303, + "mean": 0.014963302718693967 + }, + "heads90": { + "count": 4800, + "p10": 3.0, + "median": 6.0, + "p90": 9.0, + "mean": 6.310208333333334 + } + } + }, + "joint": { + "Q": { + "weak_half": { + "count": 32000, + "p10": 0.0225396528840065, + "median": 0.08576029911637306, + "p90": 0.23477475494146346, + "mean": 0.11232942398573505 + }, + "strong_half": { + "count": 32000, + "p10": 0.05655890852212906, + "median": 0.20693699270486832, + "p90": 0.5206475317478181, + "mean": 0.2704283235834446 + }, + "random_half": { + "count": 32000, + "p10": 0.039973394572734834, + "median": 0.15583113580942154, + "p90": 0.4062000095844269, + "mean": 0.21630935878241145 + }, + "all_heads": { + "count": 32000, + "p10": 0.06481254324316979, + "median": 0.22428051382303238, + "p90": 0.5543476462364201, + "mean": 0.2957394429202832 + } + }, + "K": { + "weak_half": { + "count": 32000, + "p10": 0.028515849635004996, + "median": 0.08704467490315437, + "p90": 0.23199979960918432, + "mean": 0.11572287764720386 + }, + "strong_half": { + "count": 32000, + "p10": 0.07840781956911087, + "median": 0.19813165813684464, + "p90": 0.45838963985443115, + "mean": 0.2555373763643147 + }, + "random_half": { + "count": 32000, + "p10": 0.05652601756155491, + "median": 0.1595868319272995, + "p90": 0.38752546906471264, + "mean": 0.21508053492175533 + }, + "all_heads": { + "count": 32000, + "p10": 0.08563644587993621, + "median": 0.21988031268119812, + "p90": 0.4996467530727399, + "mean": 0.2871456688197213 + } + }, + "V": { + "weak_half": { + "count": 32000, + "p10": 0.02640126012265682, + "median": 0.07635688036680222, + "p90": 0.21732283532619479, + "mean": 0.10623245530284475 + }, + "strong_half": { + "count": 32000, + "p10": 0.07369660735130311, + "median": 0.16931316256523132, + "p90": 0.4663332492113114, + "mean": 0.23960510214965325 + }, + "random_half": { + "count": 32000, + "p10": 0.05152657963335514, + "median": 0.13437538594007492, + "p90": 0.39179082512855534, + "mean": 0.20286414045983112 + }, + "all_heads": { + "count": 32000, + "p10": 0.08243615254759788, + "median": 0.19092711061239243, + "p90": 0.5185699462890625, + "mean": 0.2683242328287452 + } + }, + "QKV": { + "weak_half": { + "count": 32000, + "p10": 0.03147876150906086, + "median": 0.09034762158989906, + "p90": 0.23353940397501005, + "mean": 0.11618917352170684 + }, + "strong_half": { + "count": 32000, + "p10": 0.09105055332183838, + "median": 0.2091781497001648, + "p90": 0.48030592501163516, + "mean": 0.26540057950647317 + }, + "random_half": { + "count": 32000, + "p10": 0.06186078600585461, + "median": 0.16420399397611618, + "p90": 0.39710947275161745, + "mean": 0.2216180787065241 + }, + "all_heads": { + "count": 32000, + "p10": 0.09929505959153176, + "median": 0.22908245772123337, + "p90": 0.5173942029476166, + "mean": 0.29365509447891963 + } + }, + "JS": { + "weak_half": { + "count": 4800, + "p10": 0.0002201629729825072, + "median": 0.001413182239048183, + "p90": 0.0061596495099365866, + "mean": 0.002527013004010179 + }, + "strong_half": { + "count": 4800, + "p10": 0.004459654539823533, + "median": 0.02574532199651003, + "p90": 0.08861614987254145, + "mean": 0.038170266890164915 + }, + "random_half": { + "count": 4800, + "p10": 0.002647196548059583, + "median": 0.013814802747219801, + "p90": 0.05524100922048097, + "mean": 0.02303206111582123 + }, + "all_heads": { + "count": 4800, + "p10": 0.0049355274066329, + "median": 0.028837326914072037, + "p90": 0.10028255358338357, + "mean": 0.04305102449328388 + } + } + }, + "counts": { + "links": 92, + "texts": 4, + "layer_transitions": 23 + }, + "sample_index_key": [ + "fresh-en-1", + "fresh-en-2", + "fresh-zh-1", + "fresh-zh-2" + ], + "winner_identity_file": "winner-identities.csv" +} diff --git a/results/head-dominance/prior-official-reference-check.json b/results/head-dominance/prior-official-reference-check.json new file mode 100644 index 0000000..29ed680 --- /dev/null +++ b/results/head-dominance/prior-official-reference-check.json @@ -0,0 +1,288 @@ +{ + "passed": true, + "reference_commit": "fd3cf0140de03f7619ac95b5cfed05b711ff9e22", + "source_sha256": "4463b8a108f678c166755101d95f188179f54143305953d78147de0389b80ebc", + "torch": "2.15.0+cpu", + "numpy": "2.5.4", + "implementation_sha256": "48a2a97d0445dc6e08ac7f27a935637a7553417d0982eb94122d6cb02bbb499c", + "tests": [ + { + "test": "layer0_rmsnorm", + "shape": [ + 2, + 7, + 16 + ], + "max_abs_error": 2.384185791015625e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "layer0_mixer", + "shape": [ + 2, + 7, + 16 + ], + "max_abs_error": 1.862645149230957e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "layer0_mlp", + "shape": [ + 2, + 7, + 16 + ], + "max_abs_error": 2.9802322387695312e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "layer0_complete", + "shape": [ + 2, + 7, + 16 + ], + "max_abs_error": 2.384185791015625e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "layer1_rmsnorm", + "shape": [ + 2, + 7, + 16 + ], + "max_abs_error": 2.384185791015625e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "attention_probabilities", + "shape": [ + 2, + 2, + 7, + 7 + ], + "max_abs_error": 1.1920928955078125e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "layer1_mixer", + "shape": [ + 2, + 7, + 16 + ], + "max_abs_error": 5.21540641784668e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "layer1_mlp", + "shape": [ + 2, + 7, + 16 + ], + "max_abs_error": 4.470348358154297e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "layer1_complete", + "shape": [ + 2, + 7, + 16 + ], + "max_abs_error": 2.384185791015625e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_recurrent_gated_delta_rule_length1_output", + "shape": [ + 2, + 1, + 2, + 5 + ], + "max_abs_error": 4.470348358154297e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_recurrent_gated_delta_rule_length1_state", + "shape": [ + 2, + 2, + 4, + 5 + ], + "max_abs_error": 5.960464477539063e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_chunk_gated_delta_rule_length1_output", + "shape": [ + 2, + 1, + 2, + 5 + ], + "max_abs_error": 2.9802322387695312e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_chunk_gated_delta_rule_length1_state", + "shape": [ + 2, + 2, + 4, + 5 + ], + "max_abs_error": 5.960464477539063e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_recurrent_gated_delta_rule_length7_output", + "shape": [ + 2, + 7, + 2, + 5 + ], + "max_abs_error": 5.960464477539063e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_recurrent_gated_delta_rule_length7_state", + "shape": [ + 2, + 2, + 4, + 5 + ], + "max_abs_error": 5.960464477539063e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_chunk_gated_delta_rule_length7_output", + "shape": [ + 2, + 7, + 2, + 5 + ], + "max_abs_error": 7.450580596923828e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_chunk_gated_delta_rule_length7_state", + "shape": [ + 2, + 2, + 4, + 5 + ], + "max_abs_error": 1.7881393432617188e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_recurrent_gated_delta_rule_length33_output", + "shape": [ + 2, + 33, + 2, + 5 + ], + "max_abs_error": 1.1920928955078125e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_recurrent_gated_delta_rule_length33_state", + "shape": [ + 2, + 2, + 4, + 5 + ], + "max_abs_error": 2.9802322387695312e-08, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_chunk_gated_delta_rule_length33_output", + "shape": [ + 2, + 33, + 2, + 5 + ], + "max_abs_error": 5.364418029785156e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "torch_chunk_gated_delta_rule_length33_state", + "shape": [ + 2, + 2, + 4, + 5 + ], + "max_abs_error": 3.5762786865234375e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + }, + { + "test": "complete_two_layer_text_forward", + "shape": [ + 2, + 7, + 16 + ], + "max_abs_error": 3.5762786865234375e-07, + "atol": 2e-05, + "rtol": 2e-05, + "passed": true + } + ], + "limitations": "FP32 CPU semantics on tiny synthetic configurations; not full-checkpoint BF16 kernel equivalence" +} \ No newline at end of file diff --git a/results/head-dominance/protocol.json b/results/head-dominance/protocol.json new file mode 100644 index 0000000..c580248 --- /dev/null +++ b/results/head-dominance/protocol.json @@ -0,0 +1,48 @@ +{ + "scope": "all 23 adjacent layer transitions; source attention head contribution to next effective Q/K/V", + "samples": "4 previously authored zh/en texts, first32 tokens, descriptive", + "primary_queries": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ], + "influence": "L2 change / baseline token recipient-head L2; QKV score sqrt(mean of three squared relative changes))", + "linear_QK": "after causal convolution/SiLU and delta-rule L2 normalization; Q additionally divided by sqrt(key dim)", + "full_QK": "QK norm and partial RoPE applied; K/V repeated to query heads for indexing (not independent KV observations)", + "single_ablation": "remove complete source head projected output at all tokens; recompute source postattention normalization, MLP, residual and next input normalization", + "joint_ablation": "weak/strong/random half and all attention heads; one coherent token-varying removal mask for each recipient query head; ranking uses QKV single effects; no fitting", + "continuous": "effective heads=1/sum(p^2); fraction=effective/H; concentration=(sum(p^2)-1/H)/(1-1/H)", + "zero_total": "undefined concentration (NaN arrays), count explicitly", + "cost_budget_seconds": 7200, + "no_training": true, + "keep_all_weights_and_caches": true, + "limitations": [ + "not final LM quality or pruning proof", + "individual effects not additive; normalized scores not true contribution probabilities", + "token-varying masks are input-specific diagnostic and not a deployable fixed head subset", + "linear attention does not have an ordinary softmax probability matrix", + "Q/K/V scales handled separately; combined QKV is a declared balanced relative-effect score" + ] +} diff --git a/results/head-dominance/raw/fresh-en-1-link-00.npz b/results/head-dominance/raw/fresh-en-1-link-00.npz new file mode 100644 index 0000000..f2c7cea Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-00.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-01.npz b/results/head-dominance/raw/fresh-en-1-link-01.npz new file mode 100644 index 0000000..033331c Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-01.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-02.npz b/results/head-dominance/raw/fresh-en-1-link-02.npz new file mode 100644 index 0000000..0650e7a Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-02.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-03.npz b/results/head-dominance/raw/fresh-en-1-link-03.npz new file mode 100644 index 0000000..6995522 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-03.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-04.npz b/results/head-dominance/raw/fresh-en-1-link-04.npz new file mode 100644 index 0000000..1f592a5 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-04.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-05.npz b/results/head-dominance/raw/fresh-en-1-link-05.npz new file mode 100644 index 0000000..e2fb3b9 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-05.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-06.npz b/results/head-dominance/raw/fresh-en-1-link-06.npz new file mode 100644 index 0000000..f90711e Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-06.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-07.npz b/results/head-dominance/raw/fresh-en-1-link-07.npz new file mode 100644 index 0000000..d8499cf Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-07.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-08.npz b/results/head-dominance/raw/fresh-en-1-link-08.npz new file mode 100644 index 0000000..21ee5be Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-08.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-09.npz b/results/head-dominance/raw/fresh-en-1-link-09.npz new file mode 100644 index 0000000..6130b46 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-09.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-10.npz b/results/head-dominance/raw/fresh-en-1-link-10.npz new file mode 100644 index 0000000..0e6cc44 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-10.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-11.npz b/results/head-dominance/raw/fresh-en-1-link-11.npz new file mode 100644 index 0000000..8ce3a3c Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-11.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-12.npz b/results/head-dominance/raw/fresh-en-1-link-12.npz new file mode 100644 index 0000000..ab4dd07 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-12.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-13.npz b/results/head-dominance/raw/fresh-en-1-link-13.npz new file mode 100644 index 0000000..b14049f Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-13.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-14.npz b/results/head-dominance/raw/fresh-en-1-link-14.npz new file mode 100644 index 0000000..d981c4d Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-14.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-15.npz b/results/head-dominance/raw/fresh-en-1-link-15.npz new file mode 100644 index 0000000..e442083 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-15.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-16.npz b/results/head-dominance/raw/fresh-en-1-link-16.npz new file mode 100644 index 0000000..a26ef26 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-16.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-17.npz b/results/head-dominance/raw/fresh-en-1-link-17.npz new file mode 100644 index 0000000..35e86b7 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-17.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-18.npz b/results/head-dominance/raw/fresh-en-1-link-18.npz new file mode 100644 index 0000000..343bab9 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-18.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-19.npz b/results/head-dominance/raw/fresh-en-1-link-19.npz new file mode 100644 index 0000000..1b1a45b Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-19.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-20.npz b/results/head-dominance/raw/fresh-en-1-link-20.npz new file mode 100644 index 0000000..53c35f5 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-20.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-21.npz b/results/head-dominance/raw/fresh-en-1-link-21.npz new file mode 100644 index 0000000..9da4780 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-21.npz differ diff --git a/results/head-dominance/raw/fresh-en-1-link-22.npz b/results/head-dominance/raw/fresh-en-1-link-22.npz new file mode 100644 index 0000000..884c530 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-1-link-22.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-00.npz b/results/head-dominance/raw/fresh-en-2-link-00.npz new file mode 100644 index 0000000..6cde654 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-00.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-01.npz b/results/head-dominance/raw/fresh-en-2-link-01.npz new file mode 100644 index 0000000..c5a1eab Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-01.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-02.npz b/results/head-dominance/raw/fresh-en-2-link-02.npz new file mode 100644 index 0000000..49258ef Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-02.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-03.npz b/results/head-dominance/raw/fresh-en-2-link-03.npz new file mode 100644 index 0000000..7731bab Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-03.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-04.npz b/results/head-dominance/raw/fresh-en-2-link-04.npz new file mode 100644 index 0000000..288ab02 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-04.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-05.npz b/results/head-dominance/raw/fresh-en-2-link-05.npz new file mode 100644 index 0000000..522e8eb Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-05.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-06.npz b/results/head-dominance/raw/fresh-en-2-link-06.npz new file mode 100644 index 0000000..956a88f Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-06.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-07.npz b/results/head-dominance/raw/fresh-en-2-link-07.npz new file mode 100644 index 0000000..f4a653a Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-07.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-08.npz b/results/head-dominance/raw/fresh-en-2-link-08.npz new file mode 100644 index 0000000..200a477 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-08.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-09.npz b/results/head-dominance/raw/fresh-en-2-link-09.npz new file mode 100644 index 0000000..50c38b6 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-09.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-10.npz b/results/head-dominance/raw/fresh-en-2-link-10.npz new file mode 100644 index 0000000..1fed741 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-10.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-11.npz b/results/head-dominance/raw/fresh-en-2-link-11.npz new file mode 100644 index 0000000..dfd71b1 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-11.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-12.npz b/results/head-dominance/raw/fresh-en-2-link-12.npz new file mode 100644 index 0000000..adae03d Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-12.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-13.npz b/results/head-dominance/raw/fresh-en-2-link-13.npz new file mode 100644 index 0000000..93a4f68 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-13.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-14.npz b/results/head-dominance/raw/fresh-en-2-link-14.npz new file mode 100644 index 0000000..d8f0ab9 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-14.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-15.npz b/results/head-dominance/raw/fresh-en-2-link-15.npz new file mode 100644 index 0000000..e848cc0 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-15.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-16.npz b/results/head-dominance/raw/fresh-en-2-link-16.npz new file mode 100644 index 0000000..7833bb6 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-16.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-17.npz b/results/head-dominance/raw/fresh-en-2-link-17.npz new file mode 100644 index 0000000..da45cdf Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-17.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-18.npz b/results/head-dominance/raw/fresh-en-2-link-18.npz new file mode 100644 index 0000000..2c2735e Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-18.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-19.npz b/results/head-dominance/raw/fresh-en-2-link-19.npz new file mode 100644 index 0000000..ba223b7 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-19.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-20.npz b/results/head-dominance/raw/fresh-en-2-link-20.npz new file mode 100644 index 0000000..71422f3 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-20.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-21.npz b/results/head-dominance/raw/fresh-en-2-link-21.npz new file mode 100644 index 0000000..6e4a4e4 Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-21.npz differ diff --git a/results/head-dominance/raw/fresh-en-2-link-22.npz b/results/head-dominance/raw/fresh-en-2-link-22.npz new file mode 100644 index 0000000..b2b7cfe Binary files /dev/null and b/results/head-dominance/raw/fresh-en-2-link-22.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-00.npz b/results/head-dominance/raw/fresh-zh-1-link-00.npz new file mode 100644 index 0000000..9d8aa32 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-00.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-01.npz b/results/head-dominance/raw/fresh-zh-1-link-01.npz new file mode 100644 index 0000000..7cc0574 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-01.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-02.npz b/results/head-dominance/raw/fresh-zh-1-link-02.npz new file mode 100644 index 0000000..25ed784 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-02.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-03.npz b/results/head-dominance/raw/fresh-zh-1-link-03.npz new file mode 100644 index 0000000..c3f0c91 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-03.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-04.npz b/results/head-dominance/raw/fresh-zh-1-link-04.npz new file mode 100644 index 0000000..219ca0c Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-04.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-05.npz b/results/head-dominance/raw/fresh-zh-1-link-05.npz new file mode 100644 index 0000000..c14c64b Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-05.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-06.npz b/results/head-dominance/raw/fresh-zh-1-link-06.npz new file mode 100644 index 0000000..ac41c9e Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-06.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-07.npz b/results/head-dominance/raw/fresh-zh-1-link-07.npz new file mode 100644 index 0000000..00c63ab Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-07.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-08.npz b/results/head-dominance/raw/fresh-zh-1-link-08.npz new file mode 100644 index 0000000..9e2fba5 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-08.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-09.npz b/results/head-dominance/raw/fresh-zh-1-link-09.npz new file mode 100644 index 0000000..5a043ec Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-09.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-10.npz b/results/head-dominance/raw/fresh-zh-1-link-10.npz new file mode 100644 index 0000000..837b560 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-10.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-11.npz b/results/head-dominance/raw/fresh-zh-1-link-11.npz new file mode 100644 index 0000000..64de4d0 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-11.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-12.npz b/results/head-dominance/raw/fresh-zh-1-link-12.npz new file mode 100644 index 0000000..28ba138 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-12.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-13.npz b/results/head-dominance/raw/fresh-zh-1-link-13.npz new file mode 100644 index 0000000..8985d7a Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-13.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-14.npz b/results/head-dominance/raw/fresh-zh-1-link-14.npz new file mode 100644 index 0000000..fb6ab1f Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-14.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-15.npz b/results/head-dominance/raw/fresh-zh-1-link-15.npz new file mode 100644 index 0000000..76007ec Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-15.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-16.npz b/results/head-dominance/raw/fresh-zh-1-link-16.npz new file mode 100644 index 0000000..9cc3c44 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-16.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-17.npz b/results/head-dominance/raw/fresh-zh-1-link-17.npz new file mode 100644 index 0000000..c158b33 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-17.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-18.npz b/results/head-dominance/raw/fresh-zh-1-link-18.npz new file mode 100644 index 0000000..3583cad Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-18.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-19.npz b/results/head-dominance/raw/fresh-zh-1-link-19.npz new file mode 100644 index 0000000..f66ff78 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-19.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-20.npz b/results/head-dominance/raw/fresh-zh-1-link-20.npz new file mode 100644 index 0000000..5ee26db Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-20.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-21.npz b/results/head-dominance/raw/fresh-zh-1-link-21.npz new file mode 100644 index 0000000..2ae07b4 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-21.npz differ diff --git a/results/head-dominance/raw/fresh-zh-1-link-22.npz b/results/head-dominance/raw/fresh-zh-1-link-22.npz new file mode 100644 index 0000000..878cca9 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-1-link-22.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-00.npz b/results/head-dominance/raw/fresh-zh-2-link-00.npz new file mode 100644 index 0000000..536a578 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-00.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-01.npz b/results/head-dominance/raw/fresh-zh-2-link-01.npz new file mode 100644 index 0000000..e74befe Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-01.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-02.npz b/results/head-dominance/raw/fresh-zh-2-link-02.npz new file mode 100644 index 0000000..2eb2c7f Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-02.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-03.npz b/results/head-dominance/raw/fresh-zh-2-link-03.npz new file mode 100644 index 0000000..170ea4a Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-03.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-04.npz b/results/head-dominance/raw/fresh-zh-2-link-04.npz new file mode 100644 index 0000000..4a06e13 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-04.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-05.npz b/results/head-dominance/raw/fresh-zh-2-link-05.npz new file mode 100644 index 0000000..c24e230 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-05.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-06.npz b/results/head-dominance/raw/fresh-zh-2-link-06.npz new file mode 100644 index 0000000..07f7c51 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-06.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-07.npz b/results/head-dominance/raw/fresh-zh-2-link-07.npz new file mode 100644 index 0000000..46f40b8 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-07.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-08.npz b/results/head-dominance/raw/fresh-zh-2-link-08.npz new file mode 100644 index 0000000..8236088 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-08.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-09.npz b/results/head-dominance/raw/fresh-zh-2-link-09.npz new file mode 100644 index 0000000..fb3f2bf Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-09.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-10.npz b/results/head-dominance/raw/fresh-zh-2-link-10.npz new file mode 100644 index 0000000..313a7e0 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-10.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-11.npz b/results/head-dominance/raw/fresh-zh-2-link-11.npz new file mode 100644 index 0000000..cd19554 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-11.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-12.npz b/results/head-dominance/raw/fresh-zh-2-link-12.npz new file mode 100644 index 0000000..e251a93 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-12.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-13.npz b/results/head-dominance/raw/fresh-zh-2-link-13.npz new file mode 100644 index 0000000..7c36fdc Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-13.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-14.npz b/results/head-dominance/raw/fresh-zh-2-link-14.npz new file mode 100644 index 0000000..d3899b5 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-14.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-15.npz b/results/head-dominance/raw/fresh-zh-2-link-15.npz new file mode 100644 index 0000000..dc9c7af Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-15.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-16.npz b/results/head-dominance/raw/fresh-zh-2-link-16.npz new file mode 100644 index 0000000..b982507 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-16.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-17.npz b/results/head-dominance/raw/fresh-zh-2-link-17.npz new file mode 100644 index 0000000..cd31167 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-17.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-18.npz b/results/head-dominance/raw/fresh-zh-2-link-18.npz new file mode 100644 index 0000000..ed8c905 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-18.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-19.npz b/results/head-dominance/raw/fresh-zh-2-link-19.npz new file mode 100644 index 0000000..bc07504 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-19.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-20.npz b/results/head-dominance/raw/fresh-zh-2-link-20.npz new file mode 100644 index 0000000..8cd47ff Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-20.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-21.npz b/results/head-dominance/raw/fresh-zh-2-link-21.npz new file mode 100644 index 0000000..db9da9d Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-21.npz differ diff --git a/results/head-dominance/raw/fresh-zh-2-link-22.npz b/results/head-dominance/raw/fresh-zh-2-link-22.npz new file mode 100644 index 0000000..fac8310 Binary files /dev/null and b/results/head-dominance/raw/fresh-zh-2-link-22.npz differ diff --git a/results/head-dominance/report-output.txt b/results/head-dominance/report-output.txt new file mode 100644 index 0000000..c4ccab3 --- /dev/null +++ b/results/head-dominance/report-output.txt @@ -0,0 +1,49 @@ +# 上游注意力头对下一层 Q/K/V 的影响集中度 + +实测全部23个相邻层连接、4条既有中英文文本各32 token。主统计取query7–31,按文本/token/接收query头分别计算,再汇总。不是独立样本的显著性检验。 + +影响是移除上游一个头后,下一层Q/K/V向量的相对L2变化;重新计算中间MLP/残差/归一化。各头影响归一化仅构成消融影响分布,不是天然可相加的贡献概率。 + +| 对象 | 有效头比例中位数 [P10,P90] | 最大头份额中位数 | 前25%头份额中位数 | 最小25%头份额中位数 | 90%影响所需头数中位数 | +|---|---:|---:|---:|---:|---:| +| Q | 0.736 [0.506,0.861] | 18.6% | 46.0% | 10.0% | 12.0 | +| K | 0.749 [0.517,0.879] | 18.4% | 45.2% | 10.3% | 12.0 | +| V | 0.723 [0.479,0.862] | 19.0% | 46.6% | 9.5% | 12.0 | +| QKV | 0.760 [0.531,0.878] | 18.1% | 44.6% | 10.6% | 12.0 | +| JS | 0.244 [0.117,0.434] | 42.3% | 82.3% | 1.1% | 6.0 | +| sqrtJS | 0.558 [0.362,0.730] | 21.7% | 56.5% | 6.8% | 11.0 | + +sqrtJS为JS距离尺度敏感性分析:在初次完整汇总后增加,原始消融和原JS主结果不变。JS散度近似平方尺度,归一化后会放大强弱差距;JS影响份额不是注意力概率质量份额。 + +按接收层类型分层(避免总体分布混合不同机制): + +| 接收层类型 | Q有效头比例中位数 | K | V | QKV | +|---|---:|---:|---:|---:| +| linear_attention | 0.747 | 0.773 | 0.736 | 0.778 | +| full_attention | 0.677 | 0.562 | 0.629 | 0.645 | + +QKV是三个相对变化平方的均值再开根号,避免混加不同Q/K/V尺度;JS仅普通注意力接收层有。普通注意力GQA的K/V复制到query头索引用于配对,不是额外独立KV头。 + +| 联合移除 | QKV相对影响中位数 [P10,P90] | 检索JS中位数 [P10,P90] | +|---|---:|---:| +| weak_half | 0.09035 [0.03148,0.23354] | 0.001413 [0.000220,0.006160] | +| strong_half | 0.20918 [0.09105,0.48031] | 0.025745 [0.004460,0.088616] | +| random_half | 0.16420 [0.06186,0.39711] | 0.013815 [0.002647,0.055241] | +| all_heads | 0.22908 [0.09930,0.51739] | 0.028837 [0.004936,0.100283] | + +配对观测:QKV弱半数的实际联合影响小于强半数的比例为95.6%;强/弱影响比中位数为2.358。这是逐观测比值的中位数,不是两个总体中位数之比。弱效应为零的观测另计,不做除法。 + +联合实验的弱/强/随机半数头,在各token、各接收头上分别选择,形成完整序列再重算。排名依据单头QKV影响,同样用于JS;它是输入自适应诊断,不是免费可用的剪枝策略。线性注意力的卷积和普通注意力的QK可能传播跨token影响。所有上游头移除仍保留残差背景,不能把它称为删除全部上游信息。 + +连续指标:N_eff=1/Σp²;有效头比例=N_eff/H;C=(Σp²−1/H)/(1−1/H)。全零总影响时集中度未定义,原始数组保留NaN并在统计时排除、另计数量。单头相对L2影响在做头间占比归一化之前的总和及最大值均保留,避免把微小效应中的集中误认作重要。字段max_absolute指最大无符号相对效应,并非隐藏坐标中的未归一化L2;total是单头效应之和,真正全头联合效果另测。 + +联合实验中保存的单头影响之和仅为按query掩码加权的参考。单头消融移除整段序列的一个头,联合掩码则可跨位置变化;二者的差不能单独解释成纯非线性交互,因为时间支持也不同。应直接比较实际弱/强/随机联合效果。 + +## 文件与限制 + +`raw/*.npz`保留每次单头影响、联合影响、相应单头影响之和、删除掩码及空操作对照;`observations.npz`保留逐观测连续指标及索引;CSV和JSON为全部图表源表。原模型权重和大基线缓存继续保留在服务器。 + +只测短文本上的局部下一层作用;不代表全LM质量、可直接删头、所有输入普遍集中,或相似头可互相替代。所有头贡献可能是广泛分布的;报告不以发现稀疏为成功条件。 + +CPU测量耗时2636.4秒,峰值RSS155.9 MiB;无训练、GPU或清理。 + diff --git a/results/head-dominance/run-log.txt b/results/head-dominance/run-log.txt new file mode 100644 index 0000000..a06f983 --- /dev/null +++ b/results/head-dominance/run-log.txt @@ -0,0 +1,188 @@ +baseline layer 0 +baseline layer 1 +baseline layer 2 +baseline layer 3 +baseline layer 4 +baseline layer 5 +baseline layer 6 +baseline layer 7 +baseline layer 8 +baseline layer 9 +baseline layer 10 +baseline layer 11 +baseline layer 12 +baseline layer 13 +baseline layer 14 +baseline layer 15 +baseline layer 16 +baseline layer 17 +baseline layer 18 +baseline layer 19 +baseline layer 20 +baseline layer 21 +baseline layer 22 +baseline layer 23 +completed fresh-zh-1 0 seconds 32.8 +completed fresh-zh-1 1 seconds 28.67 +completed fresh-zh-1 2 seconds 17.99 +completed fresh-zh-1 3 seconds 26.89 +completed fresh-zh-1 4 seconds 28.66 +completed fresh-zh-1 5 seconds 27.48 +completed fresh-zh-1 6 seconds 17.4 +completed fresh-zh-1 7 seconds 26.56 +completed fresh-zh-1 8 seconds 28.43 +completed fresh-zh-1 9 seconds 29.76 +completed fresh-zh-1 10 seconds 18.37 +completed fresh-zh-1 11 seconds 26.58 +completed fresh-zh-1 12 seconds 27.76 +completed fresh-zh-1 13 seconds 26.74 +completed fresh-zh-1 14 seconds 18.09 +completed fresh-zh-1 15 seconds 35.15 +completed fresh-zh-1 16 seconds 30.58 +completed fresh-zh-1 17 seconds 29.09 +completed fresh-zh-1 18 seconds 16.57 +completed fresh-zh-1 19 seconds 26.46 +completed fresh-zh-1 20 seconds 31.07 +completed fresh-zh-1 21 seconds 27.98 +completed fresh-zh-1 22 seconds 18.65 +baseline layer 0 +baseline layer 1 +baseline layer 2 +baseline layer 3 +baseline layer 4 +baseline layer 5 +baseline layer 6 +baseline layer 7 +baseline layer 8 +baseline layer 9 +baseline layer 10 +baseline layer 11 +baseline layer 12 +baseline layer 13 +baseline layer 14 +baseline layer 15 +baseline layer 16 +baseline layer 17 +baseline layer 18 +baseline layer 19 +baseline layer 20 +baseline layer 21 +baseline layer 22 +baseline layer 23 +completed fresh-zh-2 0 seconds 32.02 +completed fresh-zh-2 1 seconds 28.92 +completed fresh-zh-2 2 seconds 17.74 +completed fresh-zh-2 3 seconds 26.73 +completed fresh-zh-2 4 seconds 29.51 +completed fresh-zh-2 5 seconds 28.23 +completed fresh-zh-2 6 seconds 16.78 +completed fresh-zh-2 7 seconds 27.55 +completed fresh-zh-2 8 seconds 31.11 +completed fresh-zh-2 9 seconds 28.51 +completed fresh-zh-2 10 seconds 19.15 +completed fresh-zh-2 11 seconds 25.65 +completed fresh-zh-2 12 seconds 32.44 +completed fresh-zh-2 13 seconds 32.48 +completed fresh-zh-2 14 seconds 18.19 +completed fresh-zh-2 15 seconds 27.91 +completed fresh-zh-2 16 seconds 28.89 +completed fresh-zh-2 17 seconds 27.96 +completed fresh-zh-2 18 seconds 18.43 +completed fresh-zh-2 19 seconds 27.23 +completed fresh-zh-2 20 seconds 31.86 +completed fresh-zh-2 21 seconds 30.71 +completed fresh-zh-2 22 seconds 17.53 +baseline layer 0 +baseline layer 1 +baseline layer 2 +baseline layer 3 +baseline layer 4 +baseline layer 5 +baseline layer 6 +baseline layer 7 +baseline layer 8 +baseline layer 9 +baseline layer 10 +baseline layer 11 +baseline layer 12 +baseline layer 13 +baseline layer 14 +baseline layer 15 +baseline layer 16 +baseline layer 17 +baseline layer 18 +baseline layer 19 +baseline layer 20 +baseline layer 21 +baseline layer 22 +baseline layer 23 +completed fresh-en-1 0 seconds 31.04 +completed fresh-en-1 1 seconds 28.07 +completed fresh-en-1 2 seconds 17.32 +completed fresh-en-1 3 seconds 24.98 +completed fresh-en-1 4 seconds 28.32 +completed fresh-en-1 5 seconds 29.88 +completed fresh-en-1 6 seconds 16.23 +completed fresh-en-1 7 seconds 24.91 +completed fresh-en-1 8 seconds 28.81 +completed fresh-en-1 9 seconds 33.2 +completed fresh-en-1 10 seconds 20.11 +completed fresh-en-1 11 seconds 28.21 +completed fresh-en-1 12 seconds 29.09 +completed fresh-en-1 13 seconds 28.87 +completed fresh-en-1 14 seconds 17.27 +completed fresh-en-1 15 seconds 27.75 +completed fresh-en-1 16 seconds 28.77 +completed fresh-en-1 17 seconds 28.91 +completed fresh-en-1 18 seconds 16.5 +completed fresh-en-1 19 seconds 25.84 +completed fresh-en-1 20 seconds 27.17 +completed fresh-en-1 21 seconds 27.81 +completed fresh-en-1 22 seconds 16.05 +baseline layer 0 +baseline layer 1 +baseline layer 2 +baseline layer 3 +baseline layer 4 +baseline layer 5 +baseline layer 6 +baseline layer 7 +baseline layer 8 +baseline layer 9 +baseline layer 10 +baseline layer 11 +baseline layer 12 +baseline layer 13 +baseline layer 14 +baseline layer 15 +baseline layer 16 +baseline layer 17 +baseline layer 18 +baseline layer 19 +baseline layer 20 +baseline layer 21 +baseline layer 22 +baseline layer 23 +completed fresh-en-2 0 seconds 33.16 +completed fresh-en-2 1 seconds 28.05 +completed fresh-en-2 2 seconds 16.95 +completed fresh-en-2 3 seconds 26.46 +completed fresh-en-2 4 seconds 28.99 +completed fresh-en-2 5 seconds 28.44 +completed fresh-en-2 6 seconds 17.17 +completed fresh-en-2 7 seconds 24.41 +completed fresh-en-2 8 seconds 32.5 +completed fresh-en-2 9 seconds 27.23 +completed fresh-en-2 10 seconds 23.81 +completed fresh-en-2 11 seconds 26.33 +completed fresh-en-2 12 seconds 28.02 +completed fresh-en-2 13 seconds 28.68 +completed fresh-en-2 14 seconds 16.05 +completed fresh-en-2 15 seconds 25.38 +completed fresh-en-2 16 seconds 28.27 +completed fresh-en-2 17 seconds 28.1 +completed fresh-en-2 18 seconds 16.29 +completed fresh-en-2 19 seconds 24.92 +completed fresh-en-2 20 seconds 26.94 +completed fresh-en-2 21 seconds 28.04 +completed fresh-en-2 22 seconds 18.22 diff --git a/results/head-dominance/samples.json b/results/head-dominance/samples.json new file mode 100644 index 0000000..fc4a3ad --- /dev/null +++ b/results/head-dominance/samples.json @@ -0,0 +1,158 @@ +[ + { + "id": "fresh-zh-1", + "language": "zh", + "text": "\u690d\u7269\u5b66\u5bb6\u5728\u4e0d\u540c\u6d77\u62d4\u91c7\u96c6\u4e86\u53f6\u7247\u6837\u672c\uff0c\u5e76\u4e3a\u6bcf\u4e00\u4efd\u6807\u672c\u8bb0\u5f55\u6e29\u5ea6\u3001\u5149\u7167\u548c\u571f\u58e4\u6e7f\u5ea6\u3002\u56de\u5230\u5b9e\u9a8c\u5ba4\u540e\uff0c\u5979\u6309\u5730\u70b9\u6574\u7406\u6807\u7b7e\uff0c\u5148\u6838\u5bf9\u7f3a\u5931\u8bb0\u5f55\uff0c\u518d\u6bd4\u8f83\u4e0d\u540c\u73af\u5883\u4e2d\u7684\u53f6\u7247\u5f62\u72b6\u3002", + "token_ids": [ + 99951, + 101071, + 115802, + 111685, + 102099, + 95759, + 111583, + 109928, + 3709, + 115026, + 134176, + 116599, + 98151, + 99472, + 5205, + 114180, + 95793, + 104074, + 102454, + 1710, + 103744, + 103911, + 95946, + 3709, + 96343, + 96656, + 100560, + 99946, + 101659, + 3709, + 96218, + 113089 + ] + }, + { + "id": "fresh-zh-2", + "language": "zh", + "text": "\u5982\u679c\u4e00\u4e2a\u76d2\u5b50\u91cc\u6709\u5341\u4e8c\u679a\u84dd\u8272\u68cb\u5b50\u548c\u516b\u679a\u767d\u8272\u68cb\u5b50\uff0c\u968f\u673a\u53d6\u51fa\u4e00\u679a\u540e\u4e0d\u653e\u56de\uff0c\u7b2c\u4e8c\u6b21\u53d6\u5230\u84dd\u8272\u7684\u6982\u7387\u9700\u8981\u6839\u636e\u7b2c\u4e00\u6b21\u7684\u7ed3\u679c\u5206\u522b\u8ba1\u7b97\u3002\u8bf7\u5148\u5217\u51fa\u5404\u4e2a\u4e8b\u4ef6\uff0c\u518d\u68c0\u67e5\u5b83\u4eec\u7684\u6982\u7387\u4e4b\u548c\u3002", + "token_ids": [ + 127356, + 98996, + 98970, + 95762, + 98175, + 101184, + 104145, + 99357, + 111324, + 96702, + 101184, + 103828, + 129730, + 3709, + 102038, + 96284, + 98938, + 101184, + 95946, + 122271, + 96120, + 3709, + 109387, + 96284, + 95839, + 116981, + 104514, + 122501, + 103720, + 109146, + 99446, + 97792 + ] + }, + { + "id": "fresh-en-1", + "language": "en", + "text": "A technician recorded the vibration of a small motor before and after replacing its bearings. She used the same mounting position and sampling interval for both measurements. The final notebook included the raw traces, instrument settings, and a brief explanation of unusual observations.", + "token_ids": [ + 32, + 51715, + 12076, + 279, + 46219, + 314, + 264, + 2526, + 8604, + 1518, + 321, + 1238, + 24101, + 1141, + 33309, + 13, + 2838, + 1429, + 279, + 1788, + 32525, + 2234, + 321, + 24102, + 9581, + 364, + 2107, + 20934, + 13, + 561, + 1534, + 35935 + ] + }, + { + "id": "fresh-en-2", + "language": "en", + "text": "The function walks through a list of integers and stores each running total in a new list. An empty input should return an empty result. Before changing the implementation, compare a short example by hand and make sure negative numbers preserve the intended behavior.", + "token_ids": [ + 760, + 709, + 21780, + 1472, + 264, + 1103, + 314, + 24959, + 321, + 10220, + 1754, + 4162, + 2702, + 303, + 264, + 491, + 1103, + 13, + 1473, + 4147, + 1879, + 1220, + 460, + 449, + 4147, + 1067, + 13, + 12851, + 9722, + 279, + 7873, + 11 + ] + } +] diff --git a/results/head-dominance/source-hashes.json b/results/head-dominance/source-hashes.json new file mode 100644 index 0000000..bdd991a --- /dev/null +++ b/results/head-dominance/source-hashes.json @@ -0,0 +1,12 @@ +{ + "dominance/report.py": "d765b4bb7542d8858bf7ac12e08c0187d36bc0c17eadf47e38778a30f5ed92ca", + "dominance/core.py": "547ccd60b357dfe4ffcfb2fa5e6ddfb7cdae29e9325e6e10e6fb99dc1542158e", + "dominance/audit.py": "d87db9483f73a8082c5f242c94f5f2b79cee690f96dc502b1e15d6a7cc4cf8a0", + "dominance/run.py": "34b6067480d95522646da5918f2b2ae4d06a608f179fa76e4732319b872707a6", + "dominance/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "dominance/PROTOCOL.md": "6ecac0c86e611e6324aa946320967f460aef762e691d8941527e81a5209e3d6b", + "dominance/README.md": "1888986693b8292f29c15cafef766dd32fde6bda188a3f404287df62f20068cb", + "tests/test_dominance.py": "bbb0f84ba8b536ab5db9a8568d9ecbf26a6a463ae135efab52bb9e4a20501b9e", + "functional/model.py": "48a2a97d0445dc6e08ac7f27a935637a7553417d0982eb94122d6cb02bbb499c", + "crossqkv/fresh_samples.json": "d3f53fc98fa9d061a3df42f24fe307b693b41a9ddf74c8377dd52096663c40a9" +} diff --git a/results/head-dominance/status.json b/results/head-dominance/status.json new file mode 100644 index 0000000..60d6afb --- /dev/null +++ b/results/head-dominance/status.json @@ -0,0 +1,928 @@ +{ + "state": "completed", + "completed_links": [ + { + "sample": "fresh-zh-1", + "source": 0, + "recipient": 1, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 32.796680768951774, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 1, + "recipient": 2, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.669062617700547, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 2, + "recipient": 3, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 17.987727428786457, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 3, + "recipient": 4, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 26.88907882804051, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 4, + "recipient": 5, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.66128103621304, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 5, + "recipient": 6, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.482413977384567, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 6, + "recipient": 7, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 17.40054892282933, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 7, + "recipient": 8, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 26.55898285517469, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 8, + "recipient": 9, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.429665393196046, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 9, + "recipient": 10, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 29.755384287796915, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 10, + "recipient": 11, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 18.37400469603017, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 11, + "recipient": 12, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 26.58053196966648, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 12, + "recipient": 13, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.76224005687982, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 13, + "recipient": 14, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 26.742808513343334, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 14, + "recipient": 15, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 18.091837794985622, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 15, + "recipient": 16, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 35.15291998302564, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 16, + "recipient": 17, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 30.582506786100566, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 17, + "recipient": 18, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 29.09135568793863, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 18, + "recipient": 19, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 16.566924762912095, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 19, + "recipient": 20, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 26.462941125035286, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 20, + "recipient": 21, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 31.07414439925924, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 21, + "recipient": 22, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.981728409882635, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-1", + "source": 22, + "recipient": 23, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 18.646536350250244, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 0, + "recipient": 1, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 32.02247520117089, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 1, + "recipient": 2, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.91754521103576, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 2, + "recipient": 3, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 17.738758038263768, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 3, + "recipient": 4, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 26.729387197177857, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 4, + "recipient": 5, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 29.509812766686082, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 5, + "recipient": 6, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.23189133219421, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 6, + "recipient": 7, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 16.784115466754884, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 7, + "recipient": 8, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.546156395226717, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 8, + "recipient": 9, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 31.108038767706603, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 9, + "recipient": 10, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.506370758637786, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 10, + "recipient": 11, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 19.15228988789022, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 11, + "recipient": 12, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 25.6509050661698, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 12, + "recipient": 13, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 32.43518339470029, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 13, + "recipient": 14, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 32.483622855041176, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 14, + "recipient": 15, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 18.194162677042186, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 15, + "recipient": 16, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.91477321414277, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 16, + "recipient": 17, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.891325166914612, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 17, + "recipient": 18, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.956439428962767, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 18, + "recipient": 19, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 18.42509563593194, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 19, + "recipient": 20, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.225743858143687, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 20, + "recipient": 21, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 31.863753782119602, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 21, + "recipient": 22, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 30.70929292915389, + "null_max": 0.0 + }, + { + "sample": "fresh-zh-2", + "source": 22, + "recipient": 23, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 17.529834564309567, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 0, + "recipient": 1, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 31.035692245699465, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 1, + "recipient": 2, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.069921141956, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 2, + "recipient": 3, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 17.31564362766221, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 3, + "recipient": 4, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 24.98184947995469, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 4, + "recipient": 5, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.318609228357673, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 5, + "recipient": 6, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 29.884420589078218, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 6, + "recipient": 7, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 16.233749402686954, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 7, + "recipient": 8, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 24.90677915001288, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 8, + "recipient": 9, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.811933936085552, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 9, + "recipient": 10, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 33.20346244610846, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 10, + "recipient": 11, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 20.10931247798726, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 11, + "recipient": 12, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.2130065462552, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 12, + "recipient": 13, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 29.093061781022698, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 13, + "recipient": 14, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.87109804712236, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 14, + "recipient": 15, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 17.273784921038896, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 15, + "recipient": 16, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.748411513864994, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 16, + "recipient": 17, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.767177529633045, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 17, + "recipient": 18, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.913388620130718, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 18, + "recipient": 19, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 16.503840344958007, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 19, + "recipient": 20, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 25.83560910774395, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 20, + "recipient": 21, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.173312211874872, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 21, + "recipient": 22, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.811572755221277, + "null_max": 0.0 + }, + { + "sample": "fresh-en-1", + "source": 22, + "recipient": 23, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 16.054285536054522, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 0, + "recipient": 1, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 33.16391439177096, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 1, + "recipient": 2, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.047963777091354, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 2, + "recipient": 3, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 16.950307322200388, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 3, + "recipient": 4, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 26.457482575904578, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 4, + "recipient": 5, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.985954388976097, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 5, + "recipient": 6, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.440314901061356, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 6, + "recipient": 7, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 17.165161206852645, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 7, + "recipient": 8, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 24.406719832215458, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 8, + "recipient": 9, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 32.50074258586392, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 9, + "recipient": 10, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 27.231423981953412, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 10, + "recipient": 11, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 23.807348831091076, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 11, + "recipient": 12, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 26.32998285489157, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 12, + "recipient": 13, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.016751427203417, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 13, + "recipient": 14, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.677621484734118, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 14, + "recipient": 15, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 16.05218999600038, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 15, + "recipient": 16, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 25.378126319963485, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 16, + "recipient": 17, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.26938945800066, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 17, + "recipient": 18, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.100269113201648, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 18, + "recipient": 19, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 16.28857011720538, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 19, + "recipient": 20, + "source_heads": 8, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 24.924683273769915, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 20, + "recipient": 21, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 26.936329636722803, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 21, + "recipient": 22, + "source_heads": 16, + "recipient_query_heads": 16, + "full_attention_recipient": false, + "elapsed_seconds": 28.038186836987734, + "null_max": 0.0 + }, + { + "sample": "fresh-en-2", + "source": 22, + "recipient": 23, + "source_heads": 16, + "recipient_query_heads": 8, + "full_attention_recipient": true, + "elapsed_seconds": 18.222678856924176, + "null_max": 0.0 + } + ], + "canonical_forward_relative_error": 0.0, + "elapsed_seconds": 2636.429194326047, + "peak_rss_kib": 159664 +} diff --git a/results/head-dominance/winner-identities.csv b/results/head-dominance/winner-identities.csv new file mode 100644 index 0000000..41cf989 --- /dev/null +++ b/results/head-dominance/winner-identities.csv @@ -0,0 +1,417 @@ +sample,source,metric,most_frequent_winner_fraction,distinct_winning_heads,head_counts +fresh-zh-1,0,Q,0.8525,7,"[0, 0, 0, 341, 23, 15, 6, 0, 10, 0, 4, 1, 0, 0, 0, 0]" +fresh-zh-1,0,K,0.78,7,"[0, 0, 0, 312, 36, 19, 0, 0, 29, 0, 1, 0, 1, 0, 0, 2]" +fresh-zh-1,0,V,0.7075,8,"[0, 0, 1, 283, 23, 43, 0, 0, 43, 0, 5, 0, 0, 1, 0, 1]" +fresh-zh-1,0,QKV,0.885,4,"[0, 0, 0, 354, 22, 15, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0]" +fresh-zh-1,1,Q,0.2925,12,"[73, 0, 5, 32, 6, 31, 0, 0, 117, 6, 32, 6, 13, 0, 67, 12]" +fresh-zh-1,1,K,0.4175,9,"[61, 0, 0, 36, 3, 21, 0, 0, 167, 0, 22, 3, 0, 0, 79, 8]" +fresh-zh-1,1,V,0.305,13,"[65, 3, 8, 35, 4, 18, 0, 0, 122, 2, 42, 7, 22, 0, 61, 11]" +fresh-zh-1,1,QKV,0.38,12,"[69, 0, 4, 35, 6, 16, 0, 0, 152, 1, 25, 5, 2, 0, 72, 13]" +fresh-zh-1,2,Q,0.985,3,"[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 197, 0, 0, 0, 0]" +fresh-zh-1,2,K,1.0,1,"[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0]" +fresh-zh-1,2,V,0.96,2,"[0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 192, 0, 0, 0, 0]" +fresh-zh-1,2,QKV,1.0,1,"[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0]" +fresh-zh-1,2,JS,0.945,6,"[0, 2, 1, 0, 0, 0, 0, 4, 0, 3, 0, 189, 1, 0, 0, 0]" +fresh-zh-1,2,sqrtJS,0.945,6,"[0, 2, 1, 0, 0, 0, 0, 4, 0, 3, 0, 189, 1, 0, 0, 0]" +fresh-zh-1,3,Q,0.4425,7,"[59, 0, 3, 5, 148, 4, 4, 177]" +fresh-zh-1,3,K,0.54,7,"[83, 2, 0, 2, 91, 4, 2, 216]" +fresh-zh-1,3,V,0.6975,8,"[50, 4, 1, 6, 279, 1, 3, 56]" +fresh-zh-1,3,QKV,0.4625,5,"[66, 1, 0, 0, 185, 0, 1, 147]" +fresh-zh-1,4,Q,0.3275,15,"[26, 3, 2, 40, 6, 0, 2, 3, 1, 8, 28, 12, 131, 25, 69, 44]" +fresh-zh-1,4,K,0.405,13,"[39, 3, 0, 31, 6, 0, 1, 2, 0, 6, 15, 8, 162, 19, 76, 32]" +fresh-zh-1,4,V,0.4075,14,"[17, 0, 3, 3, 21, 16, 2, 12, 0, 4, 8, 18, 163, 15, 44, 74]" +fresh-zh-1,4,QKV,0.425,15,"[28, 2, 1, 26, 9, 0, 2, 2, 1, 2, 15, 13, 170, 23, 55, 51]" +fresh-zh-1,5,Q,0.4,14,"[33, 4, 14, 160, 1, 2, 52, 9, 7, 24, 16, 4, 0, 0, 64, 10]" +fresh-zh-1,5,K,0.4825,16,"[20, 3, 5, 193, 3, 1, 53, 25, 10, 11, 5, 10, 2, 4, 50, 5]" +fresh-zh-1,5,V,0.4025,15,"[0, 12, 1, 161, 4, 4, 16, 22, 21, 65, 8, 15, 7, 12, 11, 41]" +fresh-zh-1,5,QKV,0.5025,15,"[13, 2, 4, 201, 2, 0, 52, 11, 10, 34, 7, 7, 1, 2, 34, 20]" +fresh-zh-1,6,Q,0.99,2,"[0, 0, 2, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0]" +fresh-zh-1,6,K,0.98,2,"[0, 0, 4, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0]" +fresh-zh-1,6,V,0.94,3,"[0, 0, 4, 8, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0]" +fresh-zh-1,6,QKV,0.945,3,"[0, 0, 5, 6, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0]" +fresh-zh-1,6,JS,0.73,7,"[0, 0, 6, 4, 0, 0, 5, 12, 146, 0, 1, 0, 0, 0, 0, 26]" +fresh-zh-1,6,sqrtJS,0.73,7,"[0, 0, 6, 4, 0, 0, 5, 12, 146, 0, 1, 0, 0, 0, 0, 26]" +fresh-zh-1,7,Q,0.5625,6,"[225, 14, 46, 89, 23, 0, 3, 0]" +fresh-zh-1,7,K,0.4325,7,"[173, 59, 94, 50, 22, 1, 1, 0]" +fresh-zh-1,7,V,0.6675,6,"[267, 38, 20, 56, 16, 0, 3, 0]" +fresh-zh-1,7,QKV,0.605,6,"[242, 19, 41, 83, 13, 0, 2, 0]" +fresh-zh-1,8,Q,0.1775,13,"[71, 0, 16, 25, 54, 29, 29, 17, 10, 0, 0, 13, 14, 42, 66, 14]" +fresh-zh-1,8,K,0.345,13,"[48, 0, 2, 19, 33, 38, 138, 16, 10, 0, 0, 11, 6, 27, 48, 4]" +fresh-zh-1,8,V,0.21,16,"[84, 2, 1, 25, 42, 40, 71, 9, 15, 1, 3, 22, 13, 31, 37, 4]" +fresh-zh-1,8,QKV,0.215,13,"[74, 0, 1, 24, 55, 35, 86, 10, 11, 0, 0, 13, 8, 26, 55, 2]" +fresh-zh-1,9,Q,0.22,11,"[35, 0, 3, 39, 6, 8, 88, 0, 87, 57, 43, 13, 21, 0, 0, 0]" +fresh-zh-1,9,K,0.26,10,"[7, 1, 0, 39, 12, 0, 101, 0, 95, 104, 3, 32, 6, 0, 0, 0]" +fresh-zh-1,9,V,0.25,14,"[15, 8, 15, 37, 10, 5, 84, 0, 100, 50, 27, 10, 35, 1, 0, 3]" +fresh-zh-1,9,QKV,0.28,11,"[16, 1, 0, 35, 1, 4, 112, 0, 108, 66, 29, 13, 15, 0, 0, 0]" +fresh-zh-1,10,Q,0.755,7,"[1, 0, 16, 15, 2, 8, 0, 0, 0, 0, 7, 151, 0, 0, 0, 0]" +fresh-zh-1,10,K,0.52,4,"[0, 0, 104, 88, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0]" +fresh-zh-1,10,V,0.28,10,"[8, 0, 56, 24, 52, 32, 4, 0, 0, 0, 4, 8, 0, 0, 8, 4]" +fresh-zh-1,10,QKV,0.48,8,"[1, 0, 96, 56, 8, 14, 0, 0, 0, 0, 4, 19, 0, 0, 2, 0]" +fresh-zh-1,10,JS,0.375,8,"[0, 0, 46, 11, 54, 7, 0, 1, 0, 0, 2, 75, 0, 4, 0, 0]" +fresh-zh-1,10,sqrtJS,0.375,8,"[0, 0, 46, 11, 54, 7, 0, 1, 0, 0, 2, 75, 0, 4, 0, 0]" +fresh-zh-1,11,Q,0.9475,4,"[0, 0, 19, 1, 1, 0, 379, 0]" +fresh-zh-1,11,K,0.945,5,"[0, 0, 1, 9, 9, 0, 378, 3]" +fresh-zh-1,11,V,0.9425,5,"[1, 0, 1, 13, 8, 0, 377, 0]" +fresh-zh-1,11,QKV,0.9675,4,"[0, 0, 9, 1, 3, 0, 387, 0]" +fresh-zh-1,12,Q,0.2825,12,"[14, 64, 3, 0, 0, 0, 1, 1, 10, 113, 0, 68, 68, 24, 18, 16]" +fresh-zh-1,12,K,0.2675,12,"[9, 84, 0, 0, 0, 1, 12, 1, 9, 75, 0, 107, 52, 19, 27, 4]" +fresh-zh-1,12,V,0.155,15,"[17, 53, 12, 1, 4, 3, 36, 1, 30, 55, 0, 53, 62, 42, 20, 11]" +fresh-zh-1,12,QKV,0.2375,11,"[1, 87, 2, 0, 0, 0, 7, 0, 12, 95, 0, 75, 60, 21, 31, 9]" +fresh-zh-1,13,Q,0.36,11,"[1, 16, 71, 7, 5, 0, 36, 22, 36, 0, 0, 13, 49, 144, 0, 0]" +fresh-zh-1,13,K,0.475,11,"[0, 35, 190, 7, 22, 0, 31, 29, 31, 0, 0, 8, 10, 36, 0, 1]" +fresh-zh-1,13,V,0.415,14,"[2, 26, 166, 15, 17, 0, 47, 17, 27, 2, 0, 9, 4, 59, 5, 4]" +fresh-zh-1,13,QKV,0.4275,11,"[0, 15, 171, 7, 14, 0, 44, 14, 42, 0, 0, 10, 22, 60, 1, 0]" +fresh-zh-1,14,Q,0.69,5,"[0, 0, 0, 0, 0, 0, 29, 0, 0, 138, 0, 0, 29, 0, 1, 3]" +fresh-zh-1,14,K,0.48,3,"[0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 96, 0, 96, 0]" +fresh-zh-1,14,V,0.76,2,"[0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 152, 0]" +fresh-zh-1,14,QKV,0.545,4,"[0, 0, 0, 0, 0, 0, 6, 0, 0, 76, 0, 0, 9, 0, 109, 0]" +fresh-zh-1,14,JS,0.275,6,"[0, 0, 0, 0, 0, 0, 55, 0, 0, 42, 0, 0, 49, 1, 37, 16]" +fresh-zh-1,14,sqrtJS,0.275,6,"[0, 0, 0, 0, 0, 0, 55, 0, 0, 42, 0, 0, 49, 1, 37, 16]" +fresh-zh-1,15,Q,0.7625,6,"[0, 0, 3, 305, 25, 4, 61, 2]" +fresh-zh-1,15,K,0.6125,8,"[3, 6, 4, 245, 71, 2, 67, 2]" +fresh-zh-1,15,V,0.37,6,"[4, 19, 7, 138, 84, 0, 148, 0]" +fresh-zh-1,15,QKV,0.6875,5,"[0, 0, 2, 275, 46, 2, 75, 0]" +fresh-zh-1,16,Q,0.4075,11,"[0, 34, 31, 33, 0, 48, 7, 0, 4, 163, 0, 19, 19, 0, 7, 35]" +fresh-zh-1,16,K,0.325,10,"[0, 28, 33, 25, 0, 58, 19, 0, 4, 130, 0, 12, 10, 0, 0, 81]" +fresh-zh-1,16,V,0.4025,14,"[2, 15, 12, 15, 0, 35, 9, 0, 3, 161, 1, 2, 51, 3, 3, 88]" +fresh-zh-1,16,QKV,0.405,11,"[0, 19, 19, 27, 0, 42, 8, 0, 2, 162, 0, 6, 31, 0, 3, 81]" +fresh-zh-1,17,Q,0.9875,4,"[2, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 395, 0, 0, 0, 0]" +fresh-zh-1,17,K,0.975,4,"[7, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 390, 0, 0, 0, 0]" +fresh-zh-1,17,V,0.9825,5,"[2, 2, 0, 0, 1, 0, 0, 0, 2, 0, 0, 393, 0, 0, 0, 0]" +fresh-zh-1,17,QKV,0.995,2,"[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0]" +fresh-zh-1,18,Q,0.865,6,"[8, 4, 0, 0, 0, 10, 0, 3, 0, 0, 0, 0, 0, 0, 173, 2]" +fresh-zh-1,18,K,0.42,3,"[48, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0]" +fresh-zh-1,18,V,0.86,4,"[20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 172, 4]" +fresh-zh-1,18,QKV,0.745,5,"[32, 11, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 149, 5]" +fresh-zh-1,18,JS,0.915,4,"[183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1]" +fresh-zh-1,18,sqrtJS,0.915,4,"[183, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1]" +fresh-zh-1,19,Q,0.9725,4,"[0, 1, 0, 0, 389, 0, 4, 6]" +fresh-zh-1,19,K,0.945,4,"[0, 0, 0, 0, 378, 1, 6, 15]" +fresh-zh-1,19,V,0.8225,4,"[0, 0, 0, 0, 329, 1, 13, 57]" +fresh-zh-1,19,QKV,0.975,2,"[0, 0, 0, 0, 390, 0, 0, 10]" +fresh-zh-1,20,Q,0.7025,7,"[35, 1, 281, 0, 0, 0, 0, 0, 1, 51, 6, 25, 0, 0, 0, 0]" +fresh-zh-1,20,K,0.54,7,"[18, 1, 216, 0, 0, 0, 0, 0, 4, 111, 2, 48, 0, 0, 0, 0]" +fresh-zh-1,20,V,0.7075,6,"[40, 1, 283, 0, 1, 0, 0, 0, 0, 53, 0, 22, 0, 0, 0, 0]" +fresh-zh-1,20,QKV,0.695,5,"[28, 0, 278, 0, 0, 0, 0, 0, 1, 80, 0, 13, 0, 0, 0, 0]" +fresh-zh-1,21,Q,0.675,11,"[11, 10, 1, 0, 2, 1, 46, 270, 29, 0, 4, 1, 0, 0, 0, 25]" +fresh-zh-1,21,K,0.665,7,"[15, 38, 0, 0, 0, 0, 52, 266, 0, 0, 8, 1, 0, 0, 0, 20]" +fresh-zh-1,21,V,0.7875,13,"[2, 6, 1, 1, 1, 2, 33, 315, 7, 1, 10, 0, 0, 3, 0, 18]" +fresh-zh-1,21,QKV,0.7475,8,"[4, 19, 0, 1, 0, 0, 55, 299, 4, 0, 3, 0, 0, 0, 0, 15]" +fresh-zh-1,22,Q,0.47,6,"[0, 0, 94, 0, 0, 0, 1, 0, 0, 16, 0, 0, 1, 84, 0, 4]" +fresh-zh-1,22,K,0.88,2,"[0, 0, 176, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0]" +fresh-zh-1,22,V,0.76,3,"[0, 0, 152, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 24]" +fresh-zh-1,22,QKV,0.84,3,"[0, 0, 168, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 7]" +fresh-zh-1,22,JS,0.64,4,"[0, 0, 128, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 43, 0, 1]" +fresh-zh-1,22,sqrtJS,0.64,4,"[0, 0, 128, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 43, 0, 1]" +fresh-zh-2,0,Q,0.8075,6,"[0, 0, 0, 323, 39, 0, 21, 0, 1, 0, 0, 8, 0, 8, 0, 0]" +fresh-zh-2,0,K,0.7625,9,"[0, 0, 0, 305, 52, 9, 17, 0, 12, 1, 0, 1, 2, 1, 0, 0]" +fresh-zh-2,0,V,0.655,10,"[0, 2, 0, 262, 33, 53, 3, 0, 20, 2, 0, 2, 4, 19, 0, 0]" +fresh-zh-2,0,QKV,0.8575,8,"[0, 0, 0, 343, 38, 1, 9, 0, 4, 0, 0, 1, 2, 2, 0, 0]" +fresh-zh-2,1,Q,0.2025,12,"[60, 31, 9, 32, 0, 34, 28, 0, 81, 23, 7, 0, 4, 0, 34, 57]" +fresh-zh-2,1,K,0.2375,12,"[49, 50, 12, 25, 0, 16, 37, 0, 95, 17, 0, 2, 1, 0, 33, 63]" +fresh-zh-2,1,V,0.2075,14,"[43, 41, 12, 22, 1, 39, 27, 0, 73, 10, 21, 1, 6, 0, 21, 83]" +fresh-zh-2,1,QKV,0.23,11,"[47, 48, 8, 22, 0, 31, 27, 0, 92, 21, 2, 0, 0, 0, 29, 73]" +fresh-zh-2,2,Q,0.935,5,"[0, 0, 0, 0, 0, 0, 1, 3, 0, 4, 0, 187, 0, 5, 0, 0]" +fresh-zh-2,2,K,0.92,4,"[0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 184, 0, 8, 0, 0]" +fresh-zh-2,2,V,0.84,6,"[4, 0, 0, 4, 0, 0, 0, 8, 0, 8, 0, 168, 0, 8, 0, 0]" +fresh-zh-2,2,QKV,0.895,5,"[0, 0, 0, 4, 0, 0, 0, 4, 0, 5, 0, 179, 0, 8, 0, 0]" +fresh-zh-2,2,JS,0.935,7,"[1, 0, 2, 0, 0, 0, 0, 2, 0, 4, 0, 187, 0, 1, 3, 0]" +fresh-zh-2,2,sqrtJS,0.935,7,"[1, 0, 2, 0, 0, 0, 0, 2, 0, 4, 0, 187, 0, 1, 3, 0]" +fresh-zh-2,3,Q,0.4875,6,"[84, 0, 0, 15, 96, 7, 3, 195]" +fresh-zh-2,3,K,0.66,6,"[95, 1, 0, 14, 22, 4, 0, 264]" +fresh-zh-2,3,V,0.395,6,"[85, 1, 0, 9, 158, 4, 0, 143]" +fresh-zh-2,3,QKV,0.5575,6,"[86, 1, 0, 12, 76, 2, 0, 223]" +fresh-zh-2,4,Q,0.21,14,"[5, 7, 1, 57, 18, 0, 1, 28, 0, 27, 19, 3, 84, 72, 73, 5]" +fresh-zh-2,4,K,0.2725,11,"[5, 9, 0, 48, 10, 0, 0, 47, 0, 12, 22, 3, 109, 60, 75, 0]" +fresh-zh-2,4,V,0.185,15,"[10, 4, 2, 26, 74, 17, 2, 64, 0, 11, 14, 7, 69, 42, 47, 11]" +fresh-zh-2,4,QKV,0.23,14,"[3, 8, 1, 49, 13, 1, 0, 54, 0, 15, 15, 3, 92, 62, 81, 3]" +fresh-zh-2,5,Q,0.3425,15,"[8, 6, 7, 137, 3, 4, 62, 40, 9, 8, 10, 18, 6, 1, 81, 0]" +fresh-zh-2,5,K,0.295,15,"[5, 3, 9, 118, 2, 2, 45, 64, 3, 1, 8, 37, 10, 5, 88, 0]" +fresh-zh-2,5,V,0.41,15,"[1, 10, 7, 164, 2, 0, 12, 57, 25, 19, 9, 40, 9, 10, 34, 1]" +fresh-zh-2,5,QKV,0.34,14,"[6, 5, 2, 136, 0, 3, 41, 60, 10, 7, 8, 34, 4, 7, 77, 0]" +fresh-zh-2,6,Q,0.875,6,"[0, 0, 1, 1, 0, 19, 0, 0, 175, 1, 0, 0, 0, 0, 0, 3]" +fresh-zh-2,6,K,0.84,5,"[0, 0, 0, 4, 0, 20, 0, 0, 168, 4, 0, 0, 0, 0, 0, 4]" +fresh-zh-2,6,V,0.62,8,"[0, 0, 4, 16, 0, 36, 4, 0, 124, 8, 4, 0, 0, 0, 4, 0]" +fresh-zh-2,6,QKV,0.77,6,"[0, 0, 1, 7, 0, 26, 0, 0, 154, 8, 0, 0, 0, 0, 0, 4]" +fresh-zh-2,6,JS,0.465,8,"[0, 0, 4, 4, 0, 15, 17, 27, 93, 2, 0, 0, 0, 0, 0, 38]" +fresh-zh-2,6,sqrtJS,0.465,8,"[0, 0, 4, 4, 0, 15, 17, 27, 93, 2, 0, 0, 0, 0, 0, 38]" +fresh-zh-2,7,Q,0.4525,6,"[181, 45, 72, 92, 6, 0, 4, 0]" +fresh-zh-2,7,K,0.3875,7,"[155, 42, 153, 39, 7, 3, 1, 0]" +fresh-zh-2,7,V,0.66,6,"[264, 30, 45, 50, 5, 0, 6, 0]" +fresh-zh-2,7,QKV,0.525,6,"[210, 25, 96, 66, 2, 0, 1, 0]" +fresh-zh-2,8,Q,0.1775,14,"[71, 0, 36, 15, 21, 28, 26, 1, 5, 4, 4, 44, 27, 63, 55, 0]" +fresh-zh-2,8,K,0.26,13,"[46, 0, 12, 5, 10, 26, 104, 1, 2, 0, 10, 69, 9, 62, 44, 0]" +fresh-zh-2,8,V,0.1725,14,"[69, 0, 21, 5, 11, 32, 58, 4, 7, 5, 13, 57, 22, 62, 34, 0]" +fresh-zh-2,8,QKV,0.1975,12,"[56, 0, 5, 9, 19, 27, 47, 0, 2, 0, 9, 79, 24, 74, 49, 0]" +fresh-zh-2,9,Q,0.2325,12,"[59, 13, 0, 18, 5, 4, 58, 0, 31, 45, 93, 67, 6, 1, 0, 0]" +fresh-zh-2,9,K,0.4,11,"[36, 13, 0, 14, 2, 0, 58, 0, 24, 59, 32, 160, 0, 1, 0, 1]" +fresh-zh-2,9,V,0.1925,14,"[22, 37, 1, 25, 7, 15, 68, 0, 55, 35, 38, 77, 7, 2, 0, 11]" +fresh-zh-2,9,QKV,0.2875,11,"[39, 20, 0, 15, 5, 1, 64, 0, 30, 45, 63, 115, 3, 0, 0, 0]" +fresh-zh-2,10,Q,0.57,8,"[5, 0, 28, 16, 8, 16, 0, 2, 0, 0, 11, 114, 0, 0, 0, 0]" +fresh-zh-2,10,K,0.54,4,"[0, 0, 108, 80, 0, 8, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0]" +fresh-zh-2,10,V,0.38,11,"[24, 16, 76, 8, 28, 20, 0, 0, 0, 0, 8, 4, 4, 4, 0, 8]" +fresh-zh-2,10,QKV,0.475,8,"[8, 2, 95, 41, 15, 16, 0, 0, 0, 0, 8, 15, 0, 0, 0, 0]" +fresh-zh-2,10,JS,0.505,10,"[7, 3, 35, 3, 30, 16, 0, 2, 0, 0, 0, 101, 0, 1, 0, 2]" +fresh-zh-2,10,sqrtJS,0.505,10,"[7, 3, 35, 3, 30, 16, 0, 2, 0, 0, 0, 101, 0, 1, 0, 2]" +fresh-zh-2,11,Q,0.945,6,"[4, 1, 10, 3, 4, 0, 378, 0]" +fresh-zh-2,11,K,0.8675,6,"[6, 1, 0, 34, 9, 0, 347, 3]" +fresh-zh-2,11,V,0.91,7,"[2, 3, 1, 19, 7, 0, 364, 4]" +fresh-zh-2,11,QKV,0.98,4,"[0, 0, 2, 4, 2, 0, 392, 0]" +fresh-zh-2,12,Q,0.2725,12,"[3, 68, 1, 4, 0, 0, 2, 17, 0, 81, 0, 109, 60, 13, 18, 24]" +fresh-zh-2,12,K,0.3625,13,"[8, 74, 1, 4, 0, 3, 5, 19, 0, 89, 0, 145, 25, 8, 16, 3]" +fresh-zh-2,12,V,0.21,15,"[26, 67, 10, 12, 4, 1, 34, 13, 1, 22, 0, 84, 59, 33, 23, 11]" +fresh-zh-2,12,QKV,0.3075,12,"[2, 84, 0, 3, 0, 1, 2, 19, 0, 76, 0, 123, 58, 5, 16, 11]" +fresh-zh-2,13,Q,0.2825,12,"[0, 6, 110, 1, 5, 0, 30, 17, 47, 4, 23, 16, 28, 113, 0, 0]" +fresh-zh-2,13,K,0.535,10,"[0, 13, 214, 0, 17, 0, 31, 22, 36, 1, 24, 10, 0, 32, 0, 0]" +fresh-zh-2,13,V,0.445,14,"[1, 17, 178, 0, 13, 0, 36, 14, 40, 1, 30, 26, 1, 40, 2, 1]" +fresh-zh-2,13,QKV,0.5225,11,"[0, 6, 209, 0, 9, 0, 36, 20, 40, 1, 15, 13, 4, 47, 0, 0]" +fresh-zh-2,14,Q,0.63,9,"[0, 3, 0, 0, 0, 3, 24, 0, 0, 126, 1, 0, 31, 3, 6, 3]" +fresh-zh-2,14,K,0.54,3,"[0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 84, 0, 108, 0]" +fresh-zh-2,14,V,0.9,4,"[0, 0, 0, 0, 0, 4, 0, 0, 0, 12, 0, 0, 4, 0, 180, 0]" +fresh-zh-2,14,QKV,0.77,5,"[0, 0, 0, 0, 0, 0, 3, 0, 0, 37, 0, 0, 5, 1, 154, 0]" +fresh-zh-2,14,JS,0.255,10,"[5, 6, 0, 5, 0, 0, 51, 0, 2, 45, 0, 0, 26, 1, 51, 8]" +fresh-zh-2,14,sqrtJS,0.255,10,"[5, 6, 0, 5, 0, 0, 51, 0, 2, 45, 0, 0, 26, 1, 51, 8]" +fresh-zh-2,15,Q,0.72,7,"[4, 0, 7, 288, 57, 5, 36, 3]" +fresh-zh-2,15,K,0.605,8,"[6, 15, 5, 242, 97, 1, 31, 3]" +fresh-zh-2,15,V,0.3525,7,"[4, 34, 20, 124, 141, 0, 74, 3]" +fresh-zh-2,15,QKV,0.68,6,"[3, 10, 3, 272, 85, 0, 27, 0]" +fresh-zh-2,16,Q,0.3875,12,"[0, 34, 50, 5, 17, 7, 5, 0, 10, 155, 0, 25, 7, 0, 17, 68]" +fresh-zh-2,16,K,0.3175,14,"[0, 20, 65, 1, 8, 17, 23, 0, 7, 97, 1, 9, 9, 1, 15, 127]" +fresh-zh-2,16,V,0.3225,15,"[1, 13, 11, 1, 19, 7, 12, 0, 11, 129, 1, 3, 40, 3, 30, 119]" +fresh-zh-2,16,QKV,0.3175,14,"[0, 20, 30, 2, 8, 10, 15, 0, 9, 127, 1, 12, 22, 1, 23, 120]" +fresh-zh-2,17,Q,0.89,5,"[13, 0, 0, 0, 0, 0, 0, 28, 0, 1, 0, 356, 2, 0, 0, 0]" +fresh-zh-2,17,K,0.915,3,"[15, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 366, 0, 0, 0, 0]" +fresh-zh-2,17,V,0.9775,3,"[4, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 391, 0, 0, 0, 0]" +fresh-zh-2,17,QKV,0.9675,3,"[8, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 387, 0, 0, 0, 0]" +fresh-zh-2,18,Q,0.7,7,"[8, 0, 0, 0, 0, 33, 3, 6, 0, 0, 0, 0, 0, 7, 140, 3]" +fresh-zh-2,18,K,0.64,4,"[16, 128, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 40, 0]" +fresh-zh-2,18,V,0.9,4,"[8, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 0, 180, 0]" +fresh-zh-2,18,QKV,0.72,6,"[12, 27, 0, 0, 0, 4, 0, 11, 0, 0, 0, 0, 0, 2, 144, 0]" +fresh-zh-2,18,JS,0.76,6,"[152, 26, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 13, 0]" +fresh-zh-2,18,sqrtJS,0.76,6,"[152, 26, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 13, 0]" +fresh-zh-2,19,Q,0.94,5,"[0, 1, 0, 0, 376, 3, 2, 18]" +fresh-zh-2,19,K,0.8575,4,"[0, 0, 0, 0, 343, 7, 2, 48]" +fresh-zh-2,19,V,0.8375,4,"[0, 3, 0, 0, 335, 0, 11, 51]" +fresh-zh-2,19,QKV,0.96,3,"[0, 0, 0, 0, 384, 1, 0, 15]" +fresh-zh-2,20,Q,0.4425,9,"[105, 1, 177, 1, 0, 0, 0, 0, 33, 37, 24, 20, 0, 0, 2, 0]" +fresh-zh-2,20,K,0.45,8,"[34, 1, 180, 0, 0, 1, 0, 0, 36, 115, 11, 22, 0, 0, 0, 0]" +fresh-zh-2,20,V,0.465,8,"[102, 1, 186, 0, 0, 0, 0, 0, 23, 44, 20, 22, 0, 0, 2, 0]" +fresh-zh-2,20,QKV,0.5675,7,"[50, 1, 227, 0, 0, 0, 0, 0, 30, 71, 9, 12, 0, 0, 0, 0]" +fresh-zh-2,21,Q,0.3625,10,"[0, 10, 3, 0, 2, 0, 100, 145, 56, 0, 0, 3, 43, 0, 4, 34]" +fresh-zh-2,21,K,0.375,11,"[0, 29, 0, 0, 1, 0, 92, 150, 50, 1, 0, 3, 51, 2, 4, 17]" +fresh-zh-2,21,V,0.4375,14,"[0, 9, 1, 3, 9, 0, 83, 175, 34, 1, 1, 3, 64, 3, 1, 13]" +fresh-zh-2,21,QKV,0.405,10,"[0, 12, 1, 0, 2, 0, 96, 162, 33, 0, 0, 1, 62, 0, 2, 29]" +fresh-zh-2,22,Q,0.675,5,"[0, 2, 135, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 57, 0, 0]" +fresh-zh-2,22,K,0.92,2,"[0, 0, 184, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0]" +fresh-zh-2,22,V,0.9,3,"[0, 0, 180, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 12]" +fresh-zh-2,22,QKV,0.92,2,"[0, 0, 184, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0]" +fresh-zh-2,22,JS,0.715,4,"[0, 0, 143, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 31, 0, 2]" +fresh-zh-2,22,sqrtJS,0.715,4,"[0, 0, 143, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 31, 0, 2]" +fresh-en-1,0,Q,0.7625,7,"[0, 0, 2, 305, 56, 29, 4, 0, 3, 0, 1, 0, 0, 0, 0, 0]" +fresh-en-1,0,K,0.5925,8,"[0, 0, 0, 237, 78, 42, 0, 0, 39, 1, 0, 1, 0, 1, 0, 1]" +fresh-en-1,0,V,0.5325,10,"[0, 0, 1, 213, 33, 74, 0, 0, 59, 4, 1, 0, 0, 6, 6, 3]" +fresh-en-1,0,QKV,0.745,4,"[0, 0, 0, 298, 48, 37, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0]" +fresh-en-1,1,Q,0.2725,12,"[32, 0, 1, 40, 23, 52, 0, 0, 109, 6, 42, 25, 30, 0, 37, 3]" +fresh-en-1,1,K,0.285,13,"[25, 2, 1, 48, 31, 55, 0, 0, 114, 1, 22, 26, 13, 0, 55, 7]" +fresh-en-1,1,V,0.22,14,"[29, 1, 4, 39, 32, 51, 5, 0, 88, 5, 42, 25, 31, 0, 38, 10]" +fresh-en-1,1,QKV,0.27,12,"[31, 0, 1, 43, 33, 53, 2, 0, 108, 0, 36, 30, 10, 0, 47, 6]" +fresh-en-1,2,Q,0.985,2,"[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0]" +fresh-en-1,2,K,1.0,1,"[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0]" +fresh-en-1,2,V,0.94,4,"[0, 4, 0, 0, 4, 0, 0, 4, 0, 0, 0, 188, 0, 0, 0, 0]" +fresh-en-1,2,QKV,0.98,2,"[0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0]" +fresh-en-1,2,JS,0.985,3,"[0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 197, 0, 0, 0, 0]" +fresh-en-1,2,sqrtJS,0.985,3,"[0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 197, 0, 0, 0, 0]" +fresh-en-1,3,Q,0.37,7,"[76, 0, 1, 14, 148, 12, 4, 145]" +fresh-en-1,3,K,0.3475,7,"[100, 1, 0, 25, 119, 4, 12, 139]" +fresh-en-1,3,V,0.5825,7,"[71, 6, 0, 16, 233, 7, 2, 65]" +fresh-en-1,3,QKV,0.465,7,"[73, 2, 0, 15, 186, 6, 1, 117]" +fresh-en-1,4,Q,0.2775,13,"[35, 7, 0, 51, 12, 8, 0, 4, 0, 7, 30, 3, 111, 7, 56, 69]" +fresh-en-1,4,K,0.35,13,"[34, 13, 0, 41, 4, 4, 0, 8, 1, 3, 27, 0, 140, 6, 62, 57]" +fresh-en-1,4,V,0.31,13,"[23, 4, 0, 5, 17, 10, 6, 22, 0, 7, 13, 0, 124, 7, 45, 117]" +fresh-en-1,4,QKV,0.285,14,"[40, 10, 0, 39, 4, 9, 4, 3, 0, 5, 22, 2, 114, 3, 56, 89]" +fresh-en-1,5,Q,0.2675,13,"[32, 0, 4, 107, 0, 0, 87, 22, 17, 28, 3, 20, 2, 2, 61, 15]" +fresh-en-1,5,K,0.2775,11,"[8, 0, 0, 111, 0, 0, 108, 27, 22, 14, 0, 31, 4, 1, 68, 6]" +fresh-en-1,5,V,0.31,15,"[5, 0, 2, 124, 1, 1, 59, 17, 44, 72, 5, 26, 2, 5, 11, 26]" +fresh-en-1,5,QKV,0.305,12,"[8, 0, 1, 122, 0, 0, 100, 22, 22, 38, 1, 21, 0, 1, 50, 14]" +fresh-en-1,6,Q,0.99,3,"[0, 0, 0, 1, 0, 0, 0, 0, 198, 1, 0, 0, 0, 0, 0, 0]" +fresh-en-1,6,K,0.98,2,"[0, 0, 0, 4, 0, 0, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0]" +fresh-en-1,6,V,0.86,6,"[0, 0, 4, 8, 4, 4, 0, 0, 172, 8, 0, 0, 0, 0, 0, 0]" +fresh-en-1,6,QKV,0.945,4,"[0, 0, 0, 8, 0, 2, 0, 0, 189, 1, 0, 0, 0, 0, 0, 0]" +fresh-en-1,6,JS,0.645,10,"[0, 0, 5, 3, 2, 6, 14, 18, 129, 6, 1, 0, 0, 0, 0, 16]" +fresh-en-1,6,sqrtJS,0.645,10,"[0, 0, 5, 3, 2, 6, 14, 18, 129, 6, 1, 0, 0, 0, 0, 16]" +fresh-en-1,7,Q,0.51,5,"[204, 60, 53, 80, 0, 0, 3, 0]" +fresh-en-1,7,K,0.415,4,"[166, 81, 118, 35, 0, 0, 0, 0]" +fresh-en-1,7,V,0.685,7,"[274, 47, 38, 38, 1, 1, 1, 0]" +fresh-en-1,7,QKV,0.59,5,"[236, 39, 55, 69, 0, 0, 1, 0]" +fresh-en-1,8,Q,0.245,14,"[38, 0, 20, 17, 33, 18, 37, 13, 3, 0, 2, 2, 33, 67, 98, 19]" +fresh-en-1,8,K,0.2375,14,"[54, 0, 9, 4, 15, 18, 95, 11, 2, 0, 1, 1, 17, 65, 85, 23]" +fresh-en-1,8,V,0.19,15,"[65, 0, 5, 11, 13, 23, 73, 15, 8, 1, 1, 10, 20, 67, 76, 12]" +fresh-en-1,8,QKV,0.2775,13,"[68, 0, 3, 2, 14, 18, 69, 8, 1, 0, 0, 3, 19, 75, 111, 9]" +fresh-en-1,9,Q,0.255,14,"[32, 0, 14, 37, 1, 13, 38, 0, 9, 102, 102, 25, 7, 18, 1, 1]" +fresh-en-1,9,K,0.27,14,"[13, 4, 9, 23, 4, 2, 56, 2, 7, 108, 57, 89, 0, 23, 0, 3]" +fresh-en-1,9,V,0.1725,15,"[10, 16, 21, 35, 1, 23, 45, 6, 23, 69, 49, 27, 14, 58, 0, 3]" +fresh-en-1,9,QKV,0.2575,13,"[17, 3, 12, 33, 0, 4, 43, 0, 14, 103, 97, 38, 4, 30, 0, 2]" +fresh-en-1,10,Q,0.66,8,"[10, 0, 11, 15, 2, 25, 0, 0, 0, 1, 0, 132, 0, 4, 0, 0]" +fresh-en-1,10,K,0.54,7,"[4, 0, 108, 72, 4, 4, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0]" +fresh-en-1,10,V,0.28,9,"[20, 0, 56, 16, 28, 32, 0, 0, 0, 8, 12, 8, 0, 20, 0, 0]" +fresh-en-1,10,QKV,0.445,9,"[12, 0, 89, 16, 15, 24, 0, 0, 0, 6, 2, 29, 0, 7, 0, 0]" +fresh-en-1,10,JS,0.485,10,"[3, 0, 33, 1, 32, 14, 1, 0, 0, 1, 0, 97, 0, 16, 2, 0]" +fresh-en-1,10,sqrtJS,0.485,10,"[3, 0, 33, 1, 32, 14, 1, 0, 0, 1, 0, 97, 0, 16, 2, 0]" +fresh-en-1,11,Q,0.935,4,"[0, 0, 19, 0, 4, 0, 374, 3]" +fresh-en-1,11,K,0.88,6,"[0, 0, 1, 21, 18, 1, 352, 7]" +fresh-en-1,11,V,0.8675,7,"[6, 1, 0, 31, 5, 1, 347, 9]" +fresh-en-1,11,QKV,0.9575,6,"[1, 0, 9, 1, 5, 0, 383, 1]" +fresh-en-1,12,Q,0.42,12,"[7, 67, 0, 1, 1, 0, 17, 0, 11, 168, 0, 53, 54, 15, 1, 5]" +fresh-en-1,12,K,0.2825,13,"[8, 113, 0, 0, 9, 2, 29, 8, 17, 101, 0, 74, 21, 16, 1, 1]" +fresh-en-1,12,V,0.1825,15,"[14, 68, 6, 5, 7, 1, 73, 1, 28, 53, 0, 36, 55, 48, 2, 3]" +fresh-en-1,12,QKV,0.35,10,"[7, 81, 0, 0, 3, 0, 35, 0, 9, 140, 0, 60, 51, 9, 0, 5]" +fresh-en-1,13,Q,0.365,14,"[2, 39, 91, 5, 7, 0, 9, 29, 22, 2, 4, 4, 32, 146, 8, 0]" +fresh-en-1,13,K,0.525,9,"[0, 62, 210, 0, 15, 0, 3, 29, 28, 0, 0, 0, 1, 50, 2, 0]" +fresh-en-1,13,V,0.415,13,"[4, 62, 166, 3, 22, 0, 10, 25, 24, 1, 0, 1, 5, 70, 7, 0]" +fresh-en-1,13,QKV,0.4825,10,"[0, 48, 193, 0, 12, 0, 6, 17, 31, 0, 0, 1, 6, 83, 3, 0]" +fresh-en-1,14,Q,0.58,9,"[0, 4, 3, 0, 0, 2, 7, 0, 0, 116, 0, 0, 44, 18, 3, 3]" +fresh-en-1,14,K,0.5,4,"[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 88, 0, 100, 8]" +fresh-en-1,14,V,0.76,5,"[0, 0, 4, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4, 4, 152, 0]" +fresh-en-1,14,QKV,0.61,7,"[0, 0, 0, 0, 0, 3, 1, 0, 0, 51, 0, 0, 19, 1, 122, 3]" +fresh-en-1,14,JS,0.31,9,"[0, 0, 0, 10, 0, 1, 20, 0, 4, 46, 0, 0, 62, 10, 35, 12]" +fresh-en-1,14,sqrtJS,0.31,9,"[0, 0, 0, 10, 0, 1, 20, 0, 4, 46, 0, 0, 62, 10, 35, 12]" +fresh-en-1,15,Q,0.8375,6,"[0, 9, 14, 335, 27, 1, 14, 0]" +fresh-en-1,15,K,0.63,6,"[0, 50, 12, 252, 49, 0, 35, 2]" +fresh-en-1,15,V,0.46,6,"[2, 54, 11, 184, 81, 0, 68, 0]" +fresh-en-1,15,QKV,0.7825,5,"[0, 20, 7, 313, 44, 0, 16, 0]" +fresh-en-1,16,Q,0.39,13,"[2, 46, 29, 33, 13, 62, 0, 0, 2, 156, 2, 34, 12, 0, 1, 8]" +fresh-en-1,16,K,0.315,13,"[2, 32, 37, 24, 1, 98, 6, 0, 0, 126, 6, 23, 20, 0, 1, 24]" +fresh-en-1,16,V,0.425,13,"[2, 26, 9, 25, 5, 47, 11, 0, 0, 170, 5, 11, 74, 0, 6, 9]" +fresh-en-1,16,QKV,0.3925,11,"[1, 39, 18, 24, 0, 80, 2, 0, 0, 157, 6, 24, 38, 0, 0, 11]" +fresh-en-1,17,Q,0.9025,8,"[0, 0, 2, 2, 1, 0, 0, 29, 1, 3, 0, 361, 0, 0, 1, 0]" +fresh-en-1,17,K,0.955,4,"[0, 0, 0, 0, 1, 0, 0, 15, 0, 2, 0, 382, 0, 0, 0, 0]" +fresh-en-1,17,V,0.985,3,"[0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 394, 0, 0, 0, 0]" +fresh-en-1,17,QKV,0.985,4,"[0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 394, 0, 0, 1, 0]" +fresh-en-1,18,Q,0.7,6,"[0, 10, 0, 0, 0, 16, 0, 0, 0, 0, 0, 7, 4, 23, 140, 0]" +fresh-en-1,18,K,0.48,4,"[40, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 96, 0]" +fresh-en-1,18,V,0.72,7,"[12, 16, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 16, 144, 0]" +fresh-en-1,18,QKV,0.68,6,"[6, 19, 0, 0, 0, 7, 0, 0, 0, 0, 0, 8, 0, 24, 136, 0]" +fresh-en-1,18,JS,0.975,4,"[2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 1, 0]" +fresh-en-1,18,sqrtJS,0.975,4,"[2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 1, 0]" +fresh-en-1,19,Q,0.985,4,"[0, 3, 0, 0, 394, 0, 2, 1]" +fresh-en-1,19,K,0.88,5,"[0, 2, 0, 0, 352, 5, 11, 30]" +fresh-en-1,19,V,0.755,6,"[3, 2, 0, 0, 302, 6, 18, 69]" +fresh-en-1,19,QKV,0.975,5,"[1, 1, 0, 0, 390, 0, 4, 4]" +fresh-en-1,20,Q,0.5175,10,"[207, 0, 12, 0, 0, 4, 5, 0, 39, 22, 4, 90, 5, 12, 0, 0]" +fresh-en-1,20,K,0.3125,11,"[125, 0, 9, 0, 0, 4, 4, 1, 50, 56, 6, 119, 16, 10, 0, 0]" +fresh-en-1,20,V,0.5975,11,"[239, 0, 17, 1, 1, 14, 0, 2, 27, 14, 0, 55, 21, 9, 0, 0]" +fresh-en-1,20,QKV,0.53,10,"[212, 0, 12, 0, 0, 8, 1, 2, 38, 32, 0, 72, 14, 9, 0, 0]" +fresh-en-1,21,Q,0.2575,15,"[43, 27, 0, 1, 3, 1, 54, 75, 15, 22, 4, 9, 18, 24, 1, 103]" +fresh-en-1,21,K,0.1925,14,"[77, 55, 4, 0, 1, 0, 42, 52, 15, 16, 8, 6, 23, 29, 1, 71]" +fresh-en-1,21,V,0.26,13,"[37, 34, 2, 0, 2, 0, 62, 104, 21, 14, 5, 6, 18, 25, 0, 70]" +fresh-en-1,21,QKV,0.2025,11,"[48, 45, 0, 0, 0, 0, 51, 79, 12, 19, 5, 4, 28, 28, 0, 81]" +fresh-en-1,22,Q,0.51,7,"[0, 0, 87, 0, 0, 1, 0, 0, 1, 6, 0, 0, 2, 102, 0, 1]" +fresh-en-1,22,K,0.82,2,"[0, 0, 164, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0]" +fresh-en-1,22,V,0.74,3,"[0, 0, 148, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 16]" +fresh-en-1,22,QKV,0.795,3,"[0, 0, 159, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 15, 0, 0]" +fresh-en-1,22,JS,0.555,4,"[0, 0, 111, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 68, 0, 2]" +fresh-en-1,22,sqrtJS,0.555,4,"[0, 0, 111, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 68, 0, 2]" +fresh-en-2,0,Q,0.68,8,"[0, 0, 0, 272, 89, 21, 5, 0, 8, 1, 0, 2, 0, 2, 0, 0]" +fresh-en-2,0,K,0.62,8,"[0, 0, 0, 248, 93, 24, 2, 0, 24, 5, 0, 1, 0, 3, 0, 0]" +fresh-en-2,0,V,0.505,10,"[0, 0, 0, 202, 59, 72, 0, 0, 49, 5, 1, 0, 1, 3, 7, 1]" +fresh-en-2,0,QKV,0.685,5,"[0, 0, 0, 274, 80, 28, 0, 0, 17, 0, 0, 0, 0, 1, 0, 0]" +fresh-en-2,1,Q,0.2125,14,"[58, 0, 7, 11, 21, 85, 11, 0, 67, 12, 2, 2, 59, 8, 35, 22]" +fresh-en-2,1,K,0.19,14,"[47, 3, 2, 13, 24, 76, 23, 0, 72, 23, 0, 3, 49, 2, 42, 21]" +fresh-en-2,1,V,0.205,15,"[40, 1, 3, 20, 21, 82, 17, 0, 59, 16, 3, 2, 75, 7, 23, 31]" +fresh-en-2,1,QKV,0.2025,14,"[45, 1, 3, 11, 25, 81, 20, 0, 71, 23, 0, 1, 50, 4, 37, 28]" +fresh-en-2,2,Q,0.975,4,"[0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 195, 0, 0, 0, 2]" +fresh-en-2,2,K,0.96,3,"[0, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 192, 0, 0, 0, 0]" +fresh-en-2,2,V,0.96,2,"[0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 192, 0, 0, 0, 0]" +fresh-en-2,2,QKV,0.965,4,"[0, 2, 0, 0, 0, 0, 0, 4, 0, 1, 0, 193, 0, 0, 0, 0]" +fresh-en-2,2,JS,0.93,5,"[0, 4, 1, 0, 0, 0, 0, 0, 0, 8, 0, 186, 0, 0, 0, 1]" +fresh-en-2,2,sqrtJS,0.93,5,"[0, 4, 1, 0, 0, 0, 0, 0, 0, 8, 0, 186, 0, 0, 0, 1]" +fresh-en-2,3,Q,0.4475,8,"[36, 2, 2, 19, 145, 11, 6, 179]" +fresh-en-2,3,K,0.455,8,"[70, 7, 1, 21, 116, 1, 2, 182]" +fresh-en-2,3,V,0.6675,8,"[36, 3, 3, 11, 267, 5, 2, 73]" +fresh-en-2,3,QKV,0.4875,7,"[48, 4, 1, 16, 195, 1, 0, 135]" +fresh-en-2,4,Q,0.395,11,"[0, 0, 0, 61, 18, 8, 0, 3, 0, 1, 45, 1, 158, 4, 59, 42]" +fresh-en-2,4,K,0.5025,12,"[0, 0, 0, 34, 7, 3, 1, 11, 1, 0, 48, 2, 201, 6, 56, 30]" +fresh-en-2,4,V,0.2625,13,"[0, 0, 0, 7, 54, 11, 1, 25, 1, 1, 41, 2, 105, 5, 54, 93]" +fresh-en-2,4,QKV,0.4325,10,"[0, 0, 0, 31, 13, 7, 0, 11, 1, 0, 44, 0, 173, 2, 70, 48]" +fresh-en-2,5,Q,0.2625,14,"[33, 0, 33, 105, 0, 1, 35, 11, 1, 3, 9, 9, 15, 19, 90, 36]" +fresh-en-2,5,K,0.27,11,"[19, 0, 37, 96, 0, 0, 46, 14, 0, 0, 5, 10, 14, 22, 108, 29]" +fresh-en-2,5,V,0.355,14,"[12, 1, 22, 142, 0, 0, 17, 11, 1, 11, 12, 16, 28, 48, 41, 38]" +fresh-en-2,5,QKV,0.3025,12,"[17, 0, 38, 121, 0, 0, 30, 10, 0, 5, 7, 11, 8, 35, 81, 37]" +fresh-en-2,6,Q,0.96,5,"[0, 4, 1, 1, 0, 0, 2, 0, 192, 0, 0, 0, 0, 0, 0, 0]" +fresh-en-2,6,K,0.88,4,"[0, 8, 0, 8, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 8]" +fresh-en-2,6,V,0.64,9,"[0, 8, 12, 12, 12, 4, 0, 0, 128, 8, 0, 0, 0, 12, 0, 4]" +fresh-en-2,6,QKV,0.815,8,"[0, 5, 3, 12, 6, 0, 4, 0, 163, 0, 0, 0, 0, 4, 0, 3]" +fresh-en-2,6,JS,0.6,10,"[1, 1, 8, 0, 3, 1, 20, 16, 120, 0, 0, 0, 0, 2, 0, 28]" +fresh-en-2,6,sqrtJS,0.6,10,"[1, 1, 8, 0, 3, 1, 20, 16, 120, 0, 0, 0, 0, 2, 0, 28]" +fresh-en-2,7,Q,0.46,6,"[184, 29, 64, 83, 23, 0, 17, 0]" +fresh-en-2,7,K,0.495,7,"[198, 40, 101, 34, 8, 2, 17, 0]" +fresh-en-2,7,V,0.61,7,"[244, 29, 51, 58, 13, 2, 3, 0]" +fresh-en-2,7,QKV,0.5625,6,"[225, 18, 67, 70, 5, 0, 15, 0]" +fresh-en-2,8,Q,0.175,14,"[55, 11, 7, 16, 14, 23, 70, 17, 12, 0, 2, 12, 39, 68, 54, 0]" +fresh-en-2,8,K,0.37,14,"[45, 10, 3, 8, 1, 30, 148, 7, 13, 0, 0, 18, 24, 51, 41, 1]" +fresh-en-2,8,V,0.285,15,"[66, 11, 5, 14, 5, 28, 114, 11, 9, 0, 2, 13, 30, 58, 33, 1]" +fresh-en-2,8,QKV,0.32,12,"[57, 12, 0, 8, 2, 29, 128, 9, 9, 0, 0, 20, 30, 52, 44, 0]" +fresh-en-2,9,Q,0.2825,12,"[40, 2, 0, 32, 27, 7, 19, 1, 22, 106, 113, 21, 10, 0, 0, 0]" +fresh-en-2,9,K,0.2775,12,"[9, 1, 0, 16, 24, 6, 23, 3, 18, 111, 98, 88, 3, 0, 0, 0]" +fresh-en-2,9,V,0.195,15,"[30, 13, 4, 25, 37, 10, 32, 13, 29, 78, 66, 29, 17, 16, 0, 1]" +fresh-en-2,9,QKV,0.295,13,"[16, 1, 0, 27, 35, 5, 21, 5, 21, 105, 118, 38, 6, 2, 0, 0]" +fresh-en-2,10,Q,0.415,11,"[1, 0, 26, 2, 47, 16, 5, 0, 8, 5, 0, 83, 0, 5, 2, 0]" +fresh-en-2,10,K,0.6,5,"[0, 0, 120, 64, 4, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0]" +fresh-en-2,10,V,0.42,11,"[0, 4, 48, 4, 84, 36, 4, 4, 0, 4, 0, 4, 0, 4, 0, 4]" +fresh-en-2,10,QKV,0.485,8,"[0, 0, 97, 8, 45, 19, 4, 0, 0, 3, 0, 23, 0, 0, 1, 0]" +fresh-en-2,10,JS,0.42,10,"[0, 1, 41, 4, 84, 9, 6, 2, 0, 2, 0, 49, 0, 2, 0, 0]" +fresh-en-2,10,sqrtJS,0.42,10,"[0, 1, 41, 4, 84, 9, 6, 2, 0, 2, 0, 49, 0, 2, 0, 0]" +fresh-en-2,11,Q,0.9375,4,"[0, 0, 14, 0, 8, 0, 375, 3]" +fresh-en-2,11,K,0.895,5,"[2, 0, 0, 6, 33, 0, 358, 1]" +fresh-en-2,11,V,0.835,5,"[5, 0, 0, 14, 28, 0, 334, 19]" +fresh-en-2,11,QKV,0.94,3,"[0, 0, 7, 0, 17, 0, 376, 0]" +fresh-en-2,12,Q,0.2225,13,"[6, 89, 1, 5, 0, 1, 0, 1, 0, 86, 14, 70, 56, 23, 22, 26]" +fresh-en-2,12,K,0.25,13,"[15, 100, 0, 9, 0, 3, 0, 3, 9, 68, 12, 89, 29, 27, 27, 9]" +fresh-en-2,12,V,0.175,16,"[24, 70, 4, 22, 4, 10, 24, 1, 18, 15, 15, 51, 53, 40, 38, 11]" +fresh-en-2,12,QKV,0.2475,11,"[2, 99, 0, 7, 0, 0, 0, 0, 3, 72, 16, 78, 57, 22, 27, 17]" +fresh-en-2,13,Q,0.315,15,"[1, 29, 126, 2, 21, 4, 23, 30, 10, 28, 7, 4, 24, 90, 0, 1]" +fresh-en-2,13,K,0.605,12,"[0, 44, 242, 0, 18, 3, 6, 25, 5, 16, 2, 1, 5, 33, 0, 0]" +fresh-en-2,13,V,0.4125,15,"[1, 47, 165, 11, 30, 7, 18, 12, 5, 17, 2, 5, 7, 70, 3, 0]" +fresh-en-2,13,QKV,0.555,12,"[0, 33, 222, 0, 16, 3, 13, 13, 6, 15, 1, 3, 17, 58, 0, 0]" +fresh-en-2,14,Q,0.835,8,"[0, 2, 1, 0, 0, 0, 5, 0, 0, 167, 0, 1, 20, 2, 0, 2]" +fresh-en-2,14,K,0.5,4,"[0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 56, 0, 100, 8]" +fresh-en-2,14,V,0.72,2,"[0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 144, 0]" +fresh-en-2,14,QKV,0.49,4,"[0, 0, 0, 0, 0, 0, 1, 0, 0, 95, 0, 0, 6, 0, 98, 0]" +fresh-en-2,14,JS,0.415,11,"[0, 3, 0, 8, 0, 2, 30, 0, 4, 83, 1, 0, 17, 2, 30, 20]" +fresh-en-2,14,sqrtJS,0.415,11,"[0, 3, 0, 8, 0, 2, 30, 0, 4, 83, 1, 0, 17, 2, 30, 20]" +fresh-en-2,15,Q,0.74,7,"[4, 3, 17, 296, 26, 1, 53, 0]" +fresh-en-2,15,K,0.6075,7,"[2, 22, 23, 243, 38, 0, 70, 2]" +fresh-en-2,15,V,0.3075,7,"[2, 47, 31, 123, 96, 1, 100, 0]" +fresh-en-2,15,QKV,0.695,5,"[0, 7, 9, 278, 39, 0, 67, 0]" +fresh-en-2,16,Q,0.61,14,"[0, 28, 24, 19, 0, 51, 6, 1, 3, 244, 1, 10, 4, 1, 5, 3]" +fresh-en-2,16,K,0.3875,14,"[0, 16, 41, 7, 1, 110, 26, 0, 1, 155, 2, 1, 9, 1, 8, 22]" +fresh-en-2,16,V,0.545,13,"[2, 18, 12, 8, 6, 65, 12, 0, 3, 218, 0, 0, 36, 2, 6, 12]" +fresh-en-2,16,QKV,0.5675,13,"[0, 22, 22, 8, 1, 82, 8, 0, 0, 227, 1, 1, 15, 1, 4, 8]" +fresh-en-2,17,Q,0.8525,7,"[0, 1, 0, 0, 0, 6, 3, 34, 0, 13, 0, 341, 0, 0, 2, 0]" +fresh-en-2,17,K,0.9125,5,"[0, 0, 0, 1, 0, 3, 0, 23, 0, 8, 0, 365, 0, 0, 0, 0]" +fresh-en-2,17,V,0.965,5,"[0, 0, 0, 0, 0, 4, 0, 6, 0, 3, 0, 386, 0, 0, 1, 0]" +fresh-en-2,17,QKV,0.9675,5,"[0, 0, 0, 0, 0, 2, 0, 7, 0, 3, 0, 387, 0, 0, 1, 0]" +fresh-en-2,18,Q,0.71,9,"[15, 1, 4, 0, 0, 18, 6, 8, 0, 0, 0, 0, 0, 1, 142, 5]" +fresh-en-2,18,K,0.34,5,"[56, 64, 0, 0, 0, 8, 0, 4, 0, 0, 0, 0, 0, 0, 68, 0]" +fresh-en-2,18,V,0.68,5,"[36, 4, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 136, 8]" +fresh-en-2,18,QKV,0.66,7,"[45, 1, 0, 0, 0, 8, 2, 6, 0, 0, 0, 0, 0, 0, 132, 6]" +fresh-en-2,18,JS,0.935,5,"[8, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 187, 2, 0]" +fresh-en-2,18,sqrtJS,0.935,5,"[8, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 187, 2, 0]" +fresh-en-2,19,Q,0.9875,3,"[0, 0, 0, 0, 395, 2, 0, 3]" +fresh-en-2,19,K,0.9125,4,"[0, 0, 0, 0, 365, 2, 6, 27]" +fresh-en-2,19,V,0.795,4,"[0, 0, 0, 0, 318, 9, 8, 65]" +fresh-en-2,19,QKV,0.975,3,"[0, 0, 0, 0, 390, 1, 0, 9]" +fresh-en-2,20,Q,0.56,7,"[224, 0, 0, 0, 0, 0, 1, 0, 19, 11, 1, 142, 2, 0, 0, 0]" +fresh-en-2,20,K,0.4225,9,"[150, 0, 0, 3, 0, 3, 0, 0, 28, 38, 1, 169, 6, 0, 2, 0]" +fresh-en-2,20,V,0.6425,10,"[257, 0, 2, 1, 0, 3, 0, 0, 20, 14, 1, 91, 10, 0, 1, 0]" +fresh-en-2,20,QKV,0.5775,7,"[231, 0, 0, 0, 0, 3, 0, 0, 19, 11, 0, 130, 5, 0, 1, 0]" +fresh-en-2,21,Q,0.2225,13,"[1, 27, 16, 4, 2, 0, 87, 3, 18, 8, 0, 89, 0, 59, 2, 84]" +fresh-en-2,21,K,0.2825,10,"[0, 38, 5, 18, 0, 0, 113, 0, 30, 10, 0, 65, 0, 57, 6, 58]" +fresh-en-2,21,V,0.24,13,"[0, 31, 2, 14, 1, 0, 96, 27, 21, 6, 0, 65, 2, 72, 7, 56]" +fresh-en-2,21,QKV,0.2425,12,"[0, 35, 5, 11, 1, 0, 97, 3, 24, 6, 0, 82, 0, 65, 2, 69]" +fresh-en-2,22,Q,0.47,5,"[0, 0, 94, 0, 1, 0, 0, 0, 0, 16, 0, 0, 0, 88, 0, 1]" +fresh-en-2,22,K,0.68,2,"[0, 0, 136, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0]" +fresh-en-2,22,V,0.66,3,"[0, 0, 132, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 16]" +fresh-en-2,22,QKV,0.7,3,"[0, 0, 140, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 10, 0, 0]" +fresh-en-2,22,JS,0.79,4,"[0, 0, 158, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 34, 0, 2]" +fresh-en-2,22,sqrtJS,0.79,4,"[0, 0, 158, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 34, 0, 2]" diff --git a/tests/test_dominance.py b/tests/test_dominance.py new file mode 100644 index 0000000..ebc9ab6 --- /dev/null +++ b/tests/test_dominance.py @@ -0,0 +1,78 @@ +"""Independent algebra, mixed-layer canonical, and joint-mask intervention checks.""" +import json,struct,tempfile,unittest +from pathlib import Path +from unittest.mock import patch +import numpy as np +from dominance.core import InfluenceLM,concentration,masks_from_influences,js_rows +from dominance.run import distances,link +from functional.model import StreamingLM,delta_rule,attention +from crossqkv.core import QKVModel + + +def fixture(path,kinds): + rng=np.random.default_rng(922);h=16;nh=4;dim=4 + c={'rms_norm_eps':1e-6,'hidden_size':h,'head_dim':dim,'num_attention_heads':nh,'num_key_value_heads':2,'rope_parameters':{'rope_theta':10000000.,'partial_rotary_factor':.5},'linear_num_key_heads':2,'linear_num_value_heads':4,'linear_key_head_dim':4,'linear_value_head_dim':4,'layer_types':kinds} + a={} + def add(name,shape):a['model.language_model.'+name]=(rng.standard_normal(shape)*.15).astype(np.float32) + add('embed_tokens.weight',(19,h));add('norm.weight',(h,)) + for l,kind in enumerate(kinds): + p=f'layers.{l}.' + for k in ('input_layernorm','post_attention_layernorm'):add(p+k+'.weight',(h,)) + for k,shape in [('gate_proj',(24,h)),('up_proj',(24,h)),('down_proj',(h,24))]:add(p+'mlp.'+k+'.weight',shape) + if kind=='full_attention': + for k,shape in [('q_proj',(32,h)),('k_proj',(8,h)),('v_proj',(8,h)),('o_proj',(h,16)),('q_norm',(4,)),('k_norm',(4,))]:add(p+'self_attn.'+k+'.weight',shape) + else: + for k,shape in [('in_proj_qkv',(32,h)),('in_proj_z',(16,h)),('in_proj_a',(4,h)),('in_proj_b',(4,h)),('out_proj',(h,16)),('conv1d',(32,1,4)),('norm',(4,))]:add(p+'linear_attn.'+k+'.weight',shape) + add(p+'linear_attn.A_log',(4,));add(p+'linear_attn.dt_bias',(4,)) + header={};body=b'' + for k,v in a.items(): + raw=v.tobytes();header[k]={'shape':list(v.shape),'dtype':'F32','data_offsets':[len(body),len(body)+len(raw)]};body+=raw + j=json.dumps(header).encode();Path(path,'model.safetensors').write_bytes(struct.pack('