I-Soofi Consortium Ikhipha I-Soofi S 30B-A3B: Imodeli Yesisekelo Ye-Hybrid Mamba-Transformer MoE YesiJalimane NesiNgisi

0 0 1 minute read

I-Soofi Consortium Ikhipha I-Soofi S 30B-A3B: Imodeli Yesisekelo Ye-Hybrid Mamba-Transformer MoE YesiJalimane NesiNgisi

Inhlangano yocwaningo yaseJalimane ishicilele umbiko wokuziqeqeshela I-Soofi S 30B-A3B. Kuyimodeli eyisisekelo evulekile yesiJalimane nesiNgisi. Ukuqeqeshwa kuphele kwaze kwaba sekugcineni kuDeutsche Telekom's Industrial AI Cloud eMunich. Izisindo zokubuka kuqala ziku-Hugging Face. Kuhle ukuqaphela ukuthi phakathi kwamanye amamodeli ayisisekelo ahlolwe ngokugcwele, i-Soofi S irekhoda amaphuzu aphezulu kakhulu esiNgisi nesiJalimane.

Yini i-Soofi S 30B-A3B?

I-Soofi S iyimodeli yesisekelo ye-Mixture-of-Experts (i-MoE) eyihybrid Mamba Transformer. Ihlanganisa ~31.6B amapharamitha futhi ivule ~3.2B ithokheni ngayinye. Njengemodeli eyisisekelo, ayinakho ukushuna kweziyalezo, ukuqondanisa, noma ukushuna kokuphepha. I-KI Bundesverband iqondisa umfelandawonye, oxhaswe uMnyango Wezindaba Zomnotho Nezamandla waseJalimane. Abahlanganyeli bahlanganisa uFraunhofer IAIS, DFKI, TU Darmstadt, ellamind, kanye neMeantix Momentum.

Indlela i-architecture esebenza ngayo?

Isimangalo sokusebenza kahle siqala ngesitaki sesendlalelo. Inethiwekhi ibamba izendlalelo ezingama-52. Lezo izendlalelo ezihlanganisa ukulandelana kwe-Mamba-2 ezingu-23, izendlalelo ze-MoE eziyimbudumbudu ezingama-23, nezingu-6 zeGrouped-Query Attention (GQA). Yilezo zingqimba ze-GQA eziyisi-6 kuphela ezigcina inqolobane ye-KV. Isendlalelo ngasinye se-MoE siphethe ochwepheshe abathutha imizila abangu-128, senza kusebenze abangu-6 ngethokheni ngayinye, futhi sengeza ochwepheshe abahlanganyelwe abangu-2. Eminye imininingwane: ubukhulu bemodeli 2688, i-ReLU eyisikwele, i-RMSNorm, futhi akukho okushumekiwe kwezikhundla.

I-Soofi S yamukela idizayini yesithenjwa ye-Nemotron 3 Nano ngaphandle kokuguqulwa. Ithimba labacwaningi linikeza izizathu ezintathu zalokho kukhetha. Lokho wukuthunyelwa ezitakini ezifana ne-vLLM, ukusebenza kahle, nokulawulwa kwesayensi. Ngenxa yokuthi umgogodla ulungisiwe, i-Nemotron 3 Nano iba isisekelo esifana nesakhiwo. Iresiphi yedatha iyona kuphela ingxenye ehambayo.

'+r[0]+''+r[1].kulungisiwe(1)+'%'; box.appendChild(div); var f=div.querySelector('.bar span'); var pmx=Math.max.apply(null,PH[i].m.map(function(z){return z[1];})); setTimeout(function(){ f.style.width=(r[1]/pmx*100)+'%'; uma (r[0]==='German') f.style.background='linear-gradient(90deg,#FFD800,#FA500F)'; }, nciphisa?0:60+k*55); }); drawLR(i); shintsha usayizi (); } [].slice.call(document.querySelectorAll('.ph')).forEach(function(b){ b.addEventListener('chofoza',function(){ setPhase(+b.dataset.i); }); }); setPhase(0); /* ———- iphaneli 5: izilinganiso ———- */ var MODELS=[‘Soofi S 30B-A3B’,’Olmo 3 32B’,’Apertus 70B’,’EuroLLM 22B’,’Alia 40B’]; var B={ 'i-English aggregate':[70.1,67.3,62.4,61.2,59.0]'i-aggregate yesiJalimane':[79.1,69.2,72.8,70.6,68.4]'HumanEval':[73.8,63.0,30.2,39.3,23.8]'MBPP-DE':[84.2,70.8,50.9,59.4,45.6]'GSM8K':[86.1,80.7,65.4,25.1,65.4]'INNCLUDE-DE':[61.2,48.2,50.4,51.1,43.9]'GPQA-Diamond':[43.4,33.3,27.3,30.3,29.8]'GLP-DE':[88.8,73.0,81.2,78.2,65.4]'LBPP':[31.0,32.1,6.4,10.7,8.6] }; var sel=document.getElementById('bsel'); Object.keys(B).forEach(function(k,i){ var b=document.createElement('button'); b.className=”act”; b.textContent=k; b.dataset.k=k; if(i===0){ b.style.borderColor=”#FA500F”=”#;0}AFstyle. sel.appendChild(b); umsebenzi drawB(k){ [].slice.call(sel.children).forEach(function(b){ var on=b.dataset.k===k; b.style.borderColor=on?'#FA500F':''; b.style.color=on?'#FFAF00':''; }); var box=document.getElementById('amabha'); box.innerHTML=''; var mx=Math.max.apply(null,B[k]); MODELS.forEach(umsebenzi(m,i){ var v=B[k][i]div=document.createElement('div'); div.className=”row”+(i===0?' hi':''); div.innerHTML=''+m+''+v.toFixed(1)+''; box.appendChild(div); var f=div.querySelector('.bar span'); uma(i!==0) f.style.background='#3a3a3a'; setTimeout(function(){ f.style.width=(v/mx*100)+'%'; }, nciphisa?0:50+i*70); }); shintsha usayizi (); } sel.addEventListener('chofoza',umsebenzi(e){ if(e.target.dataset.k) drawB(e.target.dataset.k); }); drawB('i-English aggregate'); /* ———- shintsha usayizi ngokuzenzakalela ———- */ var last=0; umsebenzi resize(){ var root=document.getElementById('sf-root'); uma(!impande) ibuya; var h=root.offsetHeight+40; uma(Math.abs(h-last)>2){ last=h; zama{ parent.postMessage({sfHeight:h},'*'); }bamba(e){} } } window.addEventListener('layisha',shintsha usayizi); window.addEventListener('resize', resize); setIsikhathi(shintsha usayizi,600); shintsha usayizi (); })();

Source link

nimda 1 hour ago

0 0 1 minute read