Katta til modellari

import numpy as np
import matplotlib.pyplot as plt
 
def yoqotish(N, N_c=8.8e13, alpha=0.076):
    """Miqyos qonuni: parametr soniga qarab kutilayotgan xato."""
    return (N_c / N) ** alpha
 
parametrlar = np.logspace(6, 12, 100)      # 1M dan 1T gacha
plt.loglog(parametrlar, yoqotish(parametrlar))
plt.xlabel("parametrlar soni")
plt.ylabel("xato (loss)")
plt.grid(True, which="both", alpha=0.3)
plt.show()

def chinchilla_optimal(parametrlar):
    """Berilgan model hajmi uchun kerakli token soni."""
    return parametrlar * 20
 
for p in [1e9, 7e9, 70e9]:
    print(f"{p/1e9:>5.0f}B parametr → {chinchilla_optimal(p)/1e9:>6.0f}B token kerak")

def attention_narxi(n_token):
    """E'tibor matritsasi kataklari soni."""
    return n_token ** 2
 
for n in [512, 4096, 32_000, 200_000, 1_000_000]:
    kataklar = attention_narxi(n)
    xotira_gb = kataklar * 2 / 1024**3        # float16 = 2 bayt
    print(f"{n:>9,} token → {kataklar:>18,} katak → {xotira_gb:>8.1f} GB")

import numpy as np
 
 
def router(token_vektori, W_router, top_k=2):
    """Tokenni qaysi mutaxassislarga yuborishni hal qiladi."""
    ballar = token_vektori @ W_router              # (n_expert,)
    tanlangan = np.argsort(ballar)[::-1][:top_k]   # eng yuqori k tasi
 
    # Faqat tanlanganlar orasida normallashtiramiz
    ogirliklar = np.exp(ballar[tanlangan])
    ogirliklar = ogirliklar / ogirliklar.sum()
 
    return tanlangan, ogirliklar
 
 
np.random.seed(7)
token = np.random.randn(64)
W_router = np.random.randn(64, 8)          # 8 ta mutaxassis
 
tanlangan, ogirliklar = router(token, W_router)
print("tanlangan mutaxassislar:", tanlangan)
print("ularning ulushi:", np.round(ogirliklar, 3))

natija = client.messages.count_tokens(
    model="claude-opus-5",
    messages=[{"role": "user", "content": open("hujjat.txt").read()}],
)
print("kirish tokenlari:", natija.input_tokens)

# 2026-yil holatiga ko'ra, 1M token uchun narx (dollarda)
NARXLAR = {
    "claude-opus-5":   {"kirish": 5.00, "chiqish": 25.00},
    "claude-sonnet-5": {"kirish": 2.00, "chiqish": 10.00},
    "claude-haiku-4-5": {"kirish": 1.00, "chiqish":  5.00},
}
 
 
def xarajat(model, kirish_tokenlar, chiqish_tokenlar):
    n = NARXLAR[model]
    return (kirish_tokenlar * n["kirish"] + chiqish_tokenlar * n["chiqish"]) / 1_000_000
 
 
# Misol: kuniga 10 000 ta so'rov, har birida 2000 kirish + 500 chiqish
kunlik = 10_000
for model in NARXLAR:
    bir_sorov = xarajat(model, 2000, 500)
    print(f"{model:<18} 1 so'rov: ${bir_sorov:.5f}   oyiga: ${bir_sorov * kunlik * 30:>9,.0f}")

response = client.messages.create(
    model="claude-opus-5",
    max_tokens=1024,
    system=[
        {
            "type": "text",
            "text": uzun_hujjat,                          # masalan 50 000 token
            "cache_control": {"type": "ephemeral"},       # SHU YERGACHA keshlanadi
        }
    ],
    messages=[{"role": "user", "content": savol}],        # bu har safar o'zgaradi
)
 
print("keshdan o'qildi:", response.usage.cache_read_input_tokens)

import time
import random
import anthropic
 
logger = logging.getLogger(__name__)
 
 
def sora_ishonchli(prompt, urinishlar=4):
    """Qayta urinish bilan xavfsiz chaqiruv."""
    for i in range(urinishlar):
        try:
            return client.messages.create(
                model="claude-opus-5",
                max_tokens=4096,
                messages=[{"role": "user", "content": prompt}],
            )
 
        except anthropic.NotFoundError:
            logger.error("Model topilmadi — model nomini tekshiring")
            raise                                   # qayta urinish foydasiz
 
        except anthropic.AuthenticationError:
            logger.error("Kalit noto'g'ri yoki muddati o'tgan")
            raise                                   # qayta urinish foydasiz
 
        except anthropic.RateLimitError:
            kutish = 2**i + random.random()         # 1s, 2s, 4s, 8s + tasodif
            logger.warning("Chegara oshdi, %.1f s kutamiz", kutish)
            time.sleep(kutish)
 
        except anthropic.APIStatusError as e:
            if e.status_code >= 500:                # server tomonda — qayta urinamiz
                time.sleep(2**i + random.random())
            else:
                logger.error("So'rov xatosi: %s", e)
                raise
 
        except anthropic.APIConnectionError:
            time.sleep(2**i + random.random())      # tarmoq — qayta urinamiz
 
    raise RuntimeError("Bir necha urinishdan keyin ham muvaffaqiyatsiz")

d = 1000          # matritsa o'lchami
r = 8             # LoRA rangi
 
toliq = d * d
lora = d * r + r * d
 
print(f"To'liq fine-tuning: {toliq:>10,} parametr")
print(f"LoRA (r={r}):        {lora:>10,} parametr")
print(f"Tejash:              {toliq / lora:>10.0f} barobar")

import torch
import torch.nn as nn
 
 
class LoRALinear(nn.Module):
    """Muzlatilgan chiziqli qatlam + o'qitiladigan past rangli qo'shimcha."""
 
    def __init__(self, asl_qatlam: nn.Linear, r=8, alpha=16):
        super().__init__()
        self.asl = asl_qatlam
        self.asl.weight.requires_grad = False        # ASL OG'IRLIKLAR MUZLATILADI
        if self.asl.bias is not None:
            self.asl.bias.requires_grad = False
 
        kirish = asl_qatlam.in_features
        chiqish = asl_qatlam.out_features
 
        # A tasodifiy, B NOL — boshida qo'shimcha ta'siri nol bo'lsin
        self.A = nn.Parameter(torch.randn(r, kirish) * 0.01)
        self.B = nn.Parameter(torch.zeros(chiqish, r))
        self.masshtab = alpha / r
 
    def forward(self, x):
        return self.asl(x) + (x @ self.A.T @ self.B.T) * self.masshtab
 
 
# Sinab ko'ramiz
asl = nn.Linear(512, 512)
lora = LoRALinear(asl, r=8)
 
jami = sum(p.numel() for p in lora.parameters())
oqitiladigan = sum(p.numel() for p in lora.parameters() if p.requires_grad)
 
print(f"Jami parametr:        {jami:>8,}")
print(f"O'qitiladigan:        {oqitiladigan:>8,}")
print(f"Ulush:                {oqitiladigan / jami:>8.2%}")

from peft import LoraConfig, get_peft_model, TaskType
from transformers import AutoModelForCausalLM
 
model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.3")
 
config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,                                   # rang — katta bo'lsa ko'proq sig'im
    lora_alpha=32,                          # odatda 2 × r
    lora_dropout=0.05,
    target_modules=["q_proj", "v_proj"],    # odatda attention matritsalari
)
 
model = get_peft_model(model, config)
model.print_trainable_parameters()

def model_hajmi(parametrlar, bit):
    """Model xotirada necha GB joy egallaydi."""
    return parametrlar * bit / 8 / 1024**3
 
 
p = 7e9      # 7 milliard parametr
 
for nom, bit in [("float32", 32), ("float16", 16), ("int8", 8), ("int4", 4)]:
    print(f"{nom:<9} {bit:>2} bit → {model_hajmi(p, bit):>6.1f} GB")

from transformers import AutoModelForCausalLM, BitsAndBytesConfig
import torch
 
konfig = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,     # hisoblash aniqroq turda
    bnb_4bit_quant_type="nf4",                 # normal float 4 — eng yaxshi variant
    bnb_4bit_use_double_quant=True,            # kvantlash konstantalarini ham siqadi
)
 
model = AutoModelForCausalLM.from_pretrained(
    "mistralai/Mistral-7B-v0.3",
    quantization_config=konfig,
    device_map="auto",
)

import torch.nn.utils.prune as prune
 
# Eng kichik 30% og'irlikni nolga tenglaymiz
for nom, modul in model.named_modules():
    if isinstance(modul, nn.Linear):
        prune.l1_unstructured(modul, name="weight", amount=0.3)
        prune.remove(modul, "weight")          # o'zgarishni doimiy qilamiz

import numpy as np
 
 
def rasmni_boklarga_bol(rasm, p=16):
    """
    rasm — (H, W, C) massiv
    Natija — (N, p*p*C), ya'ni tokenlar ketma-ketligi
    """
    H, W, C = rasm.shape
    assert H % p == 0 and W % p == 0, "o'lcham kvadratchaga bo'linishi kerak"
 
    bloklar = rasm.reshape(H // p, p, W // p, p, C)
    bloklar = bloklar.transpose(0, 2, 1, 3, 4)         # (H/p, W/p, p, p, C)
    return bloklar.reshape(-1, p * p * C)
 
 
rasm = np.random.rand(224, 224, 3)
tokenlar = rasmni_boklarga_bol(rasm, p=16)
 
print("tokenlar:", tokenlar.shape)                     # (196, 768)
print("N =", (224 * 224) // (16 * 16))                 # 196

from sentence_transformers import SentenceTransformer, util
from PIL import Image
 
model = SentenceTransformer("clip-ViT-B-32")
 
rasmlar = [Image.open(f) for f in ["mushuk.jpg", "it.jpg", "mashina.jpg"]]
izohlar = ["mushuk divanda", "it bog'da yuguryapti", "ko'chada mashina"]
 
r_vek = model.encode(rasmlar, convert_to_tensor=True)
m_vek = model.encode(izohlar, convert_to_tensor=True)
 
oxshashlik = util.cos_sim(r_vek, m_vek)
print(oxshashlik.round(decimals=2))

import evaluate
 
toxicity = evaluate.load("toxicity")
natija = toxicity.compute(predictions=model_javoblari, aggregation="ratio")
print("zararli javoblar ulushi:", natija)
 
# Ikki guruh uchun javoblarni solishtirish
regard = evaluate.load("regard", "compare")
natija = regard.compute(data=guruh1_javoblari, references=guruh2_javoblari)

SAVOL = """Do'konda 23 ta olma bor edi. 20 tasini sotishdi,
keyin 6 ta olib kelishdi. Hozir nechta olma bor?"""
 
uslublar = {
    "zero-shot": SAVOL,
 
    "zero-shot CoT": SAVOL + "\n\nQadam-baqadam o'ylab ko'ring.",
 
    "few-shot": """Savol: 5 ta qalam bor edi, 2 tasi sindi, 3 ta sotib olindi. Nechta?
Javob: 5 - 2 = 3, keyin 3 + 3 = 6. Javob: 6.
 
Savol: """ + SAVOL + "\nJavob:",
 
    "CoT few-shot": """Savol: 5 ta qalam bor edi, 2 tasi sindi, 3 ta sotib olindi. Nechta?
Javob: Boshida 5 ta bor edi.
       2 tasi sindi: 5 - 2 = 3 ta qoldi.
       3 ta qo'shildi: 3 + 3 = 6 ta.
       Javob: 6.
 
Savol: """ + SAVOL + "\nJavob:",
}
 
for nom, prompt in uslublar.items():
    javob = client.messages.create(
        model="claude-opus-5",
        max_tokens=512,
        messages=[{"role": "user", "content": prompt}],
    )
    matn = "".join(b.text for b in javob.content if b.type == "text")
    print(f"\n=== {nom} ===\n{matn.strip()[:200]}")

from collections import Counter
 
javoblar = []
for _ in range(5):
    j = client.messages.create(
        model="claude-opus-5",
        max_tokens=512,
        messages=[{"role": "user", "content": SAVOL + "\n\nQadam-baqadam o'ylang."}],
    )
    matn = "".join(b.text for b in j.content if b.type == "text")
    javoblar.append(matn.strip().split()[-1])       # oxirgi so'z — javob
 
print("eng ko'p uchragan javob:", Counter(javoblar).most_common(1))